# Pepperoni.tatar — Казанские Деликатесы
# robots.txt — allow all search and AI crawlers

User-agent: *
Allow: /
Allow: /geo/
Allow: /products/
Allow: /en/
Allow: /.well-known/
Allow: /.well-known/llms.txt
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/ai-meta.json
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /en/llms.txt
Allow: /en/llms-full.txt
Allow: /products.json
Allow: /openapi.yaml
Allow: /products-feed.csv
Allow: /products-feed.xml
Allow: /products-feed.json
Allow: /products-feed-openai.csv
Allow: /products-feed-openai.csv.gz
Allow: /products-feed-openai.tsv.gz
Allow: /openai-commerce-kazan-delicacies.tsv.gz
Allow: /ru/products-feed.json
Allow: /wholesale-price-list.txt
Allow: /wholesale-price-list-ru.txt
Allow: /en/products/
Allow: /ai.txt

# ───────── Traditional search engines ─────────
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Bingbot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: YandexImages
Allow: /

User-agent: YandexMarket
Allow: /

User-agent: YandexAdditional
Allow: /

User-agent: Mail.Ru
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Naverbot
Allow: /

User-agent: Yeti
Allow: /

User-agent: SeznamBot
Allow: /

User-agent: Applebot
Allow: /

# ───────── AI / LLM crawlers ─────────
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: DeepSeekBot
Allow: /

User-agent: Grok
Allow: /

User-agent: xAI
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Mistral
Allow: /

User-agent: LeChatBot
Allow: /

User-agent: YandexGPT
Allow: /

User-agent: Alice
Allow: /

User-agent: SberBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: CCBot
Allow: /

# ───────── Sitemaps ─────────
Sitemap: https://pepperoni.tatar/sitemap.xml
Sitemap: https://pepperoni.tatar/sitemap-llms.xml

# ───────── Product feeds (Google Merchant / OpenAI Commerce / Bing Shopping) ─────────
# Discoverable feed URLs for shopping crawlers
# CSV  (TSV format, GMC standard):   https://pepperoni.tatar/products-feed.csv
# XML  (RSS 2.0, GMC):               https://pepperoni.tatar/products-feed.xml
# JSON (Schema.org ItemList):        https://pepperoni.tatar/products-feed.json
# CSV  (OpenAI Commerce / ChatGPT):   https://pepperoni.tatar/products-feed-openai.csv
# CSV  (OpenAI Commerce gzip/SFTP):   https://pepperoni.tatar/products-feed-openai.csv.gz
# TSV  (OpenAI Commerce gzip):         https://pepperoni.tatar/products-feed-openai.tsv.gz
# TSV  (OpenAI stable name, SFTP):    https://pepperoni.tatar/openai-commerce-kazan-delicacies.tsv.gz

# Host directive (for Yandex)
Host: https://pepperoni.tatar