# =========================================================
# FRAGSCENT.COM — ROBOTS.TXT
# Updated: 25 October 2026 (final, aligned with Cloudflare)
# WordPress | REHub | Pods | WooCommerce | Filter Everything
# Content Egg | Affiliate Egg | Rank Math
# LiteSpeed Cache | Cloudflare
#
# POLICY (mirrors Cloudflare; Cloudflare is the enforcer)
# - Allow search engines and the AI search/assistant
#   crawlers Cloudflare lets through (Googlebot, Bingbot,
#   Applebot, DuckDuckBot, OAI-SearchBot, ChatGPT-User,
#   Claude-SearchBot, Claude-User, Perplexity-User,
#   DuckAssistBot, Gemini via Google-Extended, etc.).
# - Keep public fragrance content, filter collection pages
#   and agent documentation crawlable.
# - Reduce crawling of private, utility, AJAX and
#   parameter-based filter URLs.
# - Block selected AI training and bulk dataset crawlers.
# - robots.txt is not access control and does not guarantee
#   indexing or AI citations.
# =========================================================

User-agent: *

# --- WordPress administration and system ----------------
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /wp-signup.php
Disallow: /xmlrpc.php
Disallow: /wp-content/upgrade/
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_uploads/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /cdn-cgi/

# --- Public WordPress REST API / Content Egg -------------
# Intentionally not blocked:
# /wp-json/
# /wp-json/wp/v2/
# /wp-json/wp-abilities/v1/abilities
# /wp-json/wp-abilities/v1/abilities/{name}/run
# /wp-json/content-egg/v1/agent-guide
# /wp-json/content-egg/v1/agent-guide?topic=...
# /wp-json/content-egg/v1/openapi
# /wp-json/content-egg/v1/abilities/{name}/run
# Optional MCP route, only if the adapter is active:
# /wp-json/content-egg/mcp
#
# API permissions and Application Password authentication
# are enforced by WordPress, not by robots.txt.

# --- Accounts, checkout and private submission pages -----
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /cashback-account/
Disallow: /contribute/my-submissions/
Disallow: /contribute/thank-you/
Disallow: /compare-products/

# --- Affiliate redirects ---------------------------------
Disallow: /go/

# --- Internal WordPress/site search ----------------------
Disallow: /search/
Disallow: /?s=
Disallow: /*?s=
Disallow: /*&s=

# --- WooCommerce AJAX endpoints --------------------------
Disallow: /*?wc-ajax=
Disallow: /*&wc-ajax=

# --- Filters and sorting (parameter-based) ---------------
# Filter Everything path URLs (/shop/family-woody/,
# /shop/with-oud/, ...) stay crawlable: the 8 SEO-rule
# collections are indexable and listed in the filter
# sitemap below; every other combination carries
# "noindex, follow", which crawlers must be able to read.
#
# WooCommerce layered nav and Pods list filters:
Disallow: /*?filter_
Disallow: /*&filter_
Disallow: /*?query_type_
Disallow: /*&query_type_
Disallow: /*?min_price=
Disallow: /*&min_price=
Disallow: /*?max_price=
Disallow: /*&max_price=
Disallow: /*?rating_filter=
Disallow: /*&rating_filter=
Disallow: /*?orderby=
Disallow: /*&orderby=
Disallow: /*?storecategory=
Disallow: /*&storecategory=
# Filter Everything price range (budget) parameters:
Disallow: /*?min_budget=
Disallow: /*&min_budget=
Disallow: /*?max_budget=
Disallow: /*&max_budget=

# --- Cart actions, previews and utility parameters -------
Disallow: /*?add-to-cart=
Disallow: /*&add-to-cart=
Disallow: /*?remove_item=
Disallow: /*&remove_item=
Disallow: /*?add_to_wishlist=
Disallow: /*&add_to_wishlist=
Disallow: /*?compareids=
Disallow: /*&compareids=
Disallow: /*?replytocom=
Disallow: /*&replytocom=
Disallow: /*?unapproved=
Disallow: /*&unapproved=
Disallow: /*?moderation-hash=
Disallow: /*&moderation-hash=
Disallow: /*?preview=
Disallow: /*&preview=
Disallow: /*?customize_changeset_uuid=
Disallow: /*&customize_changeset_uuid=

# --- Feeds and trackbacks -------------------------------
# Keep the site root feed /feed/ crawlable.
Disallow: /comments/feed/
Disallow: /*/feed/
Disallow: /*/trackback/

# --- Public resources: intentionally crawlable ----------
# Do not block:
# CSS, JavaScript, images, /wp-includes/, public assets,
# /wp-content/ assets or /wp-json/.
#
# Keep /tag/ and /dealstore/ crawlable where crawlers must
# read their page-level noindex directive.
#
# Do not blanket-block Content Egg image-proxy URLs.
# Verify actual image src URLs before adding a proxy rule.
#
# Tracking parameters (utm_*) are handled through suitable
# canonical URLs and analytics rather than blanket blocking.
#
# Pods list pagination (?pg=) stays crawlable so crawlers
# can reach every perfume, brand, note and perfumer record.


# =========================================================
# AI CRAWLERS: TRAINING OPT-OUT AND PENDING CHANGES
# Owner policy: maximum search and AI-answer visibility;
# block AI training and bulk dataset crawlers only.
# Every crawler listed below is also blocked by Cloudflare
# (AI bot policies: Training = Disallow; custom rules "AI
# Crawl Control - Block AI bots by User Agent" and "Block
# AI dataset bots (robots.txt mirror)"). Applebot-Extended
# is a usage token, not a crawler, so only robots.txt
# controls it. Cloudflare still serves /robots.txt to
# blocked bots so they can read these rules.
# Keep this file and Cloudflare in step: when a bot is
# blocked or unblocked in Cloudflare, change it here too.
# Leave Cloudflare "Bot Preference Sync" OFF, or it will
# prepend a second, duplicate set of rules to this file.
# =========================================================

# --- AI training and dataset crawlers --------------------
# Blocked in Cloudflare too (Training policy or custom rules).
# FacebookBot is Meta's model-training crawler; Facebook
# link previews use facebookexternalhit, which stays allowed.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
User-agent: Timpibot
User-agent: Novellum
User-agent: ProRataInc
# Training opt-out token (no crawler of its own, so
# Cloudflare cannot block it; robots.txt is the control).
# Google-Extended is deliberately NOT listed: Gemini may
# use the site. It does not affect Google Search indexing.
User-agent: Applebot-Extended
# Blocked by the "Block AI dataset bots" Cloudflare rule:
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: PanguBot
User-agent: Webzio-Extended
Disallow: /

# --- AI search, assistant and other bots -----------------
# Blocked by Cloudflare (AI Crawl Control toggles; the
# Training policy for PerplexityBot and YouBot). Owner
# policy favours search/answer bots: to allow one, unblock
# it in Cloudflare FIRST, then delete its line here.
User-agent: PerplexityBot
User-agent: YouBot
User-agent: Amazonbot
User-agent: Google-CloudVertexBot
User-agent: CloudflareBrowserRenderingCrawler
User-agent: PetalBot
User-agent: TikTokSpider
Disallow: /
# (Cloudflare also blocks "Anchor Browser"; it has no
# robots.txt token, so it is enforced at Cloudflare only.)

# =========================================================
# SITEMAPS
# =========================================================
# Rank Math: posts, pages, products, brands, notes,
# perfumers, perfumes, categories.
Sitemap: https://fragscent.com/sitemap_index.xml

# Filter Everything: indexable shop collections.
Sitemap: https://fragscent.com/wp-content/uploads/filter-everything/xml/filter-sitemap-index.xml