# seoskillsai.com — robots.txt
# Universal Directory & Package Manager for AI SEO Agent Skills
# Domain: https://seoskillsai.com
# Last updated: 13 October 2026
# New indexable guide (allowed by Allow: /): /guides/deepseek-harness-seo-skills

# 1. Standard Search Engine Crawlers (Google, Bing, DuckDuckGo, Yandex, Yahoo)
User-agent: *
Allow: /

# 2. Sitemaps
Sitemap: https://seoskillsai.com/sitemap-index.xml
Sitemap: https://seoskillsai.com/sitemap-0.xml

# 3. AI / LLM Attribution & Search Engines (Allowed)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Cohere-ai
Allow: /

# 4. Aggressive Bulk Scrapers & No-Attribution Bots (Blocked)
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Amazonbot
Disallow: /

# 5. Machine-Readable LLM Standards References
# LLMs.txt: https://seoskillsai.com/llms.txt
# LLMs-Full: https://seoskillsai.com/llms-full.txt