# Bay of Ideas: where ideas find the people who can build them.
#
# Search engines, AI crawlers and AI agents are all welcome to read the public
# pages: ideas, groups, the blog and the help pages.
#
# For agents: https://bayofideas.com/llms.txt is a map of the site, https://bayofideas.com/agents
# says what agents may do and how to attribute, every idea page has a Markdown
# twin (append .md), and the MCP server is at https://mcp.bayofideas.com/mcp.
# The public read API (GET only) is described in https://bayofideas.com/openapi.json.
#
# Content-Signal (contentsignals.org): search=yes, ai-input=yes, ai-train=yes.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /api/ideas
Allow: /api/stats/public
Allow: /api/blog
Allow: /api/lookup
Allow: /api/groups
Disallow: /images/profiles/
Disallow: /images/profilebackgrounds/
Disallow: /documents/
Disallow: /api/
Disallow: /home
Disallow: /my-profile
Disallow: /my-ideas
Disallow: /my-groups
Disallow: /messages
Disallow: /notifications
Disallow: /settings
Disallow: /admin
Disallow: /oauth/

# AI crawlers and agents: welcome, with the same rules. Members can switch
# "Let AI assistants read it" off for an idea; those ideas stay on the site
# for people and search engines and are listed here for AI user agents.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: Meta-ExternalFetcher
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: cohere-ai
User-agent: CCBot
User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /api/ideas
Allow: /api/stats/public
Allow: /api/blog
Allow: /api/lookup
Allow: /api/groups
Disallow: /images/profiles/
Disallow: /images/profilebackgrounds/
Disallow: /documents/
Disallow: /api/
Disallow: /home
Disallow: /my-profile
Disallow: /my-ideas
Disallow: /my-groups
Disallow: /messages
Disallow: /notifications
Disallow: /settings
Disallow: /admin
Disallow: /oauth/

# Every public idea, group and blog post.
Sitemap: https://bayofideas.com/sitemap.xml