# robots.txt - generated by _tools/seo-set-indexing.mjs
# STATE: on, AI training blocked, answer engines allowed.
#
# To block everything again:  node _tools/seo-set-indexing.mjs off
# then rebuild and push.

User-agent: *
Allow: /

# No value to a crawler, and they consume crawl budget.
Disallow: /wp-json/
Disallow: /my-account/
Disallow: /cart/
Disallow: /checkout/
Disallow: /*?utm_
Disallow: /*?replytocom=

# AI crawlers are split into two groups, because they are two different
# decisions and the earlier version of this file applied one answer to both.

# ---------------------------------------------------------------------
# TRAINING: refused. Taking the content to train a model is not something
# this file grants, and refusing it costs no visibility anywhere.
# Google-Extended covers Gemini and Vertex training only and has no effect
# on Google Search ranking.
# ---------------------------------------------------------------------

# OpenAI, training
User-agent: GPTBot
Disallow: /

# Anthropic, training
User-agent: ClaudeBot
Disallow: /

# Anthropic, legacy token
User-agent: anthropic-ai
Disallow: /

# Gemini and Vertex training; does NOT affect Google Search
User-agent: Google-Extended
Disallow: /

# Apple Intelligence training
User-agent: Applebot-Extended
Disallow: /

# Common Crawl, feeds many models
User-agent: CCBot
Disallow: /

# ByteDance
User-agent: Bytespider
Disallow: /

# Meta AI
User-agent: meta-externalagent
Disallow: /

# Meta, language models
User-agent: FacebookBot
Disallow: /

# Amazon
User-agent: Amazonbot
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# DeepSeek
User-agent: DeepSeekBot
Disallow: /

# Allen Institute
User-agent: AI2Bot
Disallow: /

# Diffbot, extraction and resale
User-agent: Diffbot
Disallow: /

# Webz.io dataset resale
User-agent: Omgilibot
Disallow: /

# image dataset collection
User-agent: ImagesiftBot
Disallow: /

# Timpi
User-agent: Timpibot
Disallow: /

# ---------------------------------------------------------------------
# ANSWER ENGINES: allowed, stated explicitly rather than left to the
# wildcard above, so the intent is legible to a human reading this file.
# These fetch a page to answer a question and cite the source. The -User
# agents fire when a person asks an assistant to look this company up.
# Refusing them does not keep anything out of a model. It only guarantees
# we cannot appear in the answer.
# ---------------------------------------------------------------------

# OpenAI, the ChatGPT Search index
User-agent: OAI-SearchBot
Allow: /

# ChatGPT fetching this page because someone asked about us
User-agent: ChatGPT-User
Allow: /

# Anthropic, search index
User-agent: Claude-SearchBot
Allow: /

# Claude fetching this page because someone asked about us
User-agent: Claude-User
Allow: /

# Perplexity, search index
User-agent: PerplexityBot
Allow: /

# Perplexity, on a user request
User-agent: Perplexity-User
Allow: /

# Mistral, on a user request
User-agent: MistralAI-User
Allow: /

# You.com, search index
User-agent: YouBot
Allow: /

Sitemap: https://www.sharepointpro.com.au/sitemap.xml