{
  "schemaVersion": "0.1",
  "id": "crawler-robots-policy-20260818",
  "generatedAt": "2026-08-18T04:04:05+08:00",
  "baseUrl": "https://lucheng-geo.pages.dev",
  "robotsPath": "/robots.txt",
  "purpose": "Make crawler access policy explicit for representative search and AI crawler User-Agents used in GEO verification.",
  "boundary": "robots policy readiness does not prove crawl execution, search indexing, AI platform ingestion, model training, answer citation, ranking, natural recommendation, or paid publishing outcome.",
  "summary": {
    "explicitCrawlerRules": 17,
    "allowsAllPublicPaths": true,
    "hasDefaultAllowAll": true,
    "hasSearchCrawlerRules": true,
    "hasAiCrawlerRules": true,
    "canClaimCrawlerPolicyL1": true,
    "canClaimSearchIndexing": false,
    "canClaimAiCrawled": false,
    "canClaimAiAdoption": false,
    "currentInterpretation": "robots.txt explicitly allows representative search and AI crawler User-Agents. This proves only crawler policy readiness, not indexing or AI adoption."
  },
  "crawlerRules": [
    {
      "userAgent": "*",
      "family": "default",
      "reason": "兜底允许所有未单独列出的公开搜索与 AI 读取代理。"
    },
    {
      "userAgent": "Googlebot",
      "family": "search-global",
      "reason": "Google 搜索发现与索引测试。"
    },
    {
      "userAgent": "Bingbot",
      "family": "search-global",
      "reason": "Bing 搜索发现与可能的 AI 搜索信源路线。"
    },
    {
      "userAgent": "Baiduspider",
      "family": "search-cn",
      "reason": "百度搜索发现测试。"
    },
    {
      "userAgent": "Sogou web spider",
      "family": "search-cn",
      "reason": "搜狗搜索发现测试。"
    },
    {
      "userAgent": "360Spider",
      "family": "search-cn",
      "reason": "360 搜索发现测试。"
    },
    {
      "userAgent": "Bytespider",
      "family": "ai-cn",
      "reason": "字节系公开抓取路线监测；不能等同豆包采用。"
    },
    {
      "userAgent": "TencentBot",
      "family": "ai-cn",
      "reason": "腾讯系公开抓取路线监测；不能等同元宝采用。"
    },
    {
      "userAgent": "OAI-SearchBot",
      "family": "ai-search",
      "reason": "OpenAI 搜索索引读取路线。"
    },
    {
      "userAgent": "GPTBot",
      "family": "ai-training",
      "reason": "OpenAI GPTBot 读取边界显式允许；不证明训练吸收。"
    },
    {
      "userAgent": "ChatGPT-User",
      "family": "ai-user",
      "reason": "用户触发式读取路线。"
    },
    {
      "userAgent": "ClaudeBot",
      "family": "ai-global",
      "reason": "Anthropic 公开 crawler 路线显式允许。"
    },
    {
      "userAgent": "PerplexityBot",
      "family": "ai-search",
      "reason": "AI 搜索产品公开 crawler 路线显式允许。"
    },
    {
      "userAgent": "Applebot",
      "family": "ai-search",
      "reason": "Apple 搜索/智能代理公开 crawler 路线显式允许。"
    },
    {
      "userAgent": "DuckDuckBot",
      "family": "search-global",
      "reason": "补充国际搜索发现入口。"
    },
    {
      "userAgent": "YandexBot",
      "family": "search-global",
      "reason": "补充国际搜索发现入口。"
    },
    {
      "userAgent": "CCBot",
      "family": "ai-common-crawl",
      "reason": "Common Crawl 路线显式允许；不证明任一具体 AI 平台采用。"
    }
  ],
  "requiredFollowUpEvidence": [
    "HTTP User-Agent access matrix after deployment.",
    "Public search result screenshot showing exact target URL or canonical equivalent.",
    "AI platform answer screenshot/transcript for direct URL reading or natural query mention.",
    "Cross-time retest evidence before claiming stable recommendation."
  ],
  "outputs": {
    "json": "data/crawler-robots-policy-20260818.json",
    "markdown": "docs/CRAWLER_ROBOTS_POLICY_20260818.md",
    "html": "reports/crawler-robots-policy-20260818.html"
  }
}