{
  "version": "2026.11.07",
  "license": {
    "id": "CC-BY-4.0",
    "url": "https://creativecommons.org/licenses/by/4.0/",
    "attribution": "Go Top SEO — https://www.gotopseo.com/en/ai-crawlers"
  },
  "source": "https://www.gotopseo.com/en/ai-crawlers",
  "dataset": "https://www.gotopseo.com/api/public/ai-crawlers",
  "measuredOn": "2026-11-07",
  "method": {
    "sample": "100 highest-ranked domains per market from the Tranco top-1M list",
    "markets": [
      ".com",
      ".de",
      ".fr",
      ".es",
      ".it",
      ".jp",
      ".nl"
    ],
    "domainsRequested": 700,
    "robotsFilesRead": 528,
    "failures": {
      "unreachable": 137,
      "servedHtmlInsteadOfRobots": 35
    },
    "how": "robots.txt fetched directly and parsed by the standard matching rules, against the token list in our public AI-crawler dataset",
    "reproducible": "Tranco is the standard academic ranking list, so the sample can be rebuilt by anyone who wants to check us. Our own site was not used to run the checks, so the measurement does not depend on our API rate limit."
  },
  "findings": {
    "ofGptbotBlockersAlsoBlockingTrafficCrawlerPct": 66.5,
    "gptbotBlockers": 161,
    "blocksAiTrafficCrawlerButAllowsGooglebotPct": 17.2,
    "blockedTrainingKeptSearchOpenPct": 10.2,
    "anyTrafficBringingCrawlerBlockedPct": 20.5,
    "byCrawler": [
      {
        "token": "GPTBot",
        "purpose": "train",
        "blockedPct": 30.5
      },
      {
        "token": "ChatGPT-User",
        "purpose": "answer",
        "blockedPct": 20.3
      },
      {
        "token": "OAI-SearchBot",
        "purpose": "search",
        "blockedPct": 16.5
      }
    ],
    "byMarket": [
      {
        "market": ".nl",
        "blocksTrafficCrawlerPct": 31.5
      },
      {
        "market": ".com",
        "blocksTrafficCrawlerPct": 29.7
      },
      {
        "market": ".fr",
        "blocksTrafficCrawlerPct": 25.8
      },
      {
        "market": ".it",
        "blocksTrafficCrawlerPct": 16.2
      },
      {
        "market": ".es",
        "blocksTrafficCrawlerPct": 15
      },
      {
        "market": ".jp",
        "blocksTrafficCrawlerPct": 13.7
      },
      {
        "market": ".de",
        "blocksTrafficCrawlerPct": 12.3
      }
    ]
  },
  "limitations": [
    "These are large, high-ranked sites, not small businesses. Among small businesses the picture is probably different — likely fewer deliberate blocks and more neglect. Do not present these figures as a picture of small businesses.",
    "n = 528 robots.txt files actually read, out of 700 domains requested. Percentages are of the files read, not of the domains requested.",
    "A robots.txt verdict states the request a site is making, not an outcome: some operators document that their agent may ignore robots.txt."
  ],
  "notSampled": {
    "markets": [
      "Brazil (.com.br)",
      "Portugal (.pt)"
    ],
    "note": "Neither was in the sample. The .com figure is the global English-language slice and must not be read as a figure for either country."
  }
}