# robots.txt for chaletdieverhuren.nl
# This file provides comprehensive guidelines to well-behaved search engine crawlers
# regarding access to specific areas of the chaletdieverhuren.nl website.
# It helps optimize crawl budget and prevents sensitive or non-essential content
# from appearing in search engine results.

User-agent: *
# This directive applies to ALL web robots (crawlers), including major search engines
# like Google, Bing, DuckDuckGo, and specialized crawlers.

Allow: /
# By default, allow all crawlers to access and index ALL content on the website.
# This is crucial for ensuring that your public-facing pages are discoverable.
# Specific disallow rules below will override this general allowance for certain paths.

# --- Disallowed Directories and Paths for Security, SEO, and Efficiency ---
# These directives prevent crawlers from accessing common administrative, temporary,
# or private areas that are not meant for public indexing. This helps to:
# 1. Enhance website security by not exposing internal structures.
# 2. Prevent duplicate content issues.
# 3. Optimize 'crawl budget' by directing crawlers to valuable content.

Disallow: /admin/                # Common administrative login areas (e.g., custom CMS admin)
Disallow: /wp-admin/             # Specific for WordPress administration panels
Disallow: /cpanel/               # Web hosting control panel access
Disallow: /includes/             # Contains core system files, often not meant for direct access
Disallow: /private/              # Directories designated for private or sensitive user data
Disallow: /temp/                 # Temporary file storage
Disallow: /tmp/                  # Another common temporary directory
Disallow: /cgi-bin/              # Common Gateway Interface scripts
Disallow: /*?                    # Crucially, disallows URLs containing ANY query parameter.
                                 # This helps prevent massive duplicate content issues often
                                 # caused by dynamic URLs (e.g., search results, filters, sessions).
                                 # Use canonical tags for specific dynamic pages you DO want indexed.

# --- Disallowed File Types (Optional - Uncomment if Applicable) ---
# Uncomment these lines if you have specific file types that are frequently found on your site
# but are NOT relevant for search engine indexing (e.g., downloadable documents, archives).
# This helps keep search results clean and relevant.

# Disallow: /*.zip$               # All .zip archive files
# Disallow: /*.rar$               # All .rar archive files
# Disallow: /*.tar.gz$            # All .tar.gz compressed archive files
# Disallow: /*.pdf$               # All PDF documents
# Disallow: /*.doc$               # All Microsoft Word documents (older format)
# Disallow: /*.docx$              # All Microsoft Word documents (newer format)
# Disallow: /*.xls$               # All Microsoft Excel spreadsheets (older format)
# Disallow: /*.xlsx$              # All Microsoft Excel spreadsheets (newer format)
# Disallow: /*.ppt$               # All Microsoft PowerPoint presentations (older format)
# Disallow: /*.pptx$              # All Microsoft PowerPoint presentations (newer format)
# Disallow: /*.csv$               # All Comma Separated Values files
# Disallow: /*.log$               # All log files

# --- Sitemap Location(s) ---
# This directive is essential. It explicitly tells search engines where to find
# the XML sitemap(s) for your website. Sitemaps are the most efficient way to
# communicate all the important pages on your site that you want crawled and indexed.
# Ensure this URL is correct and the sitemap is accessible.

Sitemap: https://www.chaletdieverhuren.nl/sitemap.xml

# If you utilize multiple sitemaps (e.g., for different content types, languages,
# or large sites), list each sitemap on a new 'Sitemap:' line:
# Sitemap: https://www.chaletdieverhuren.nl/sitemap-pages.xml
# Sitemap: https://www.chaletdieverhuren.nl/sitemap-posts.xml
# Sitemap: https://www.chaletdieverhuren.nl/sitemap-images.xml
# Sitemap: https://www.chaletdieverhuren.nl/sitemap-videos.xml

# --- Note on 'Crawl-delay' Directive ---
# The 'Crawl-delay' directive is largely outdated and NOT supported by major search engines
# such as Google, Bing, and Yahoo. These search engines use their own sophisticated,
# dynamic algorithms to determine optimal crawl rates for websites.
# Including 'Crawl-delay' is generally not recommended as it has no practical effect
# on the most important crawlers and can sometimes be misunderstood by older or less
# common bots. It has been omitted from this optimized version.
# Example of the deprecated directive: # Crawl-delay: 10