{
  "classes": [
    "training_crawler",
    "ai_search_crawler",
    "user_action_agent",
    "agentic_browser",
    "coding_agent",
    "search_engine",
    "preview_fetcher",
    "ads_checker",
    "automation_client",
    "scraper"
  ],
  "count": 26,
  "agents": [
    {
      "id": "openai-gptbot",
      "name": "GPTBot",
      "operator": "OpenAI",
      "agentClass": "training_crawler",
      "userAgentTokens": [
        "GPTBot"
      ],
      "robotsTokens": [
        "GPTBot"
      ],
      "respectsRobots": "yes",
      "verification": {
        "ipRangeUrls": [
          "https://openai.com/gptbot.json"
        ]
      },
      "purpose": "Crawls content that may be used to train OpenAI's generative AI foundation models.",
      "docsUrl": "https://developers.openai.com/api/docs/bots",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "openai-oai-searchbot",
      "name": "OAI-SearchBot",
      "operator": "OpenAI",
      "agentClass": "ai_search_crawler",
      "userAgentTokens": [
        "OAI-SearchBot"
      ],
      "robotsTokens": [
        "OAI-SearchBot"
      ],
      "respectsRobots": "yes",
      "verification": {
        "ipRangeUrls": [
          "https://openai.com/searchbot.json"
        ]
      },
      "purpose": "Surfaces websites in ChatGPT search results.",
      "docsUrl": "https://developers.openai.com/api/docs/bots",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "openai-oai-adsbot",
      "name": "OAI-AdsBot",
      "operator": "OpenAI",
      "agentClass": "ads_checker",
      "userAgentTokens": [
        "OAI-AdsBot"
      ],
      "robotsTokens": [
        "OAI-AdsBot"
      ],
      "respectsRobots": "unknown",
      "verification": {
        "ipRangeUrls": [
          "https://openai.com/adsbot.json"
        ]
      },
      "purpose": "Checks landing pages submitted as ads on ChatGPT.",
      "docsUrl": "https://developers.openai.com/api/docs/bots",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "openai-chatgpt-user",
      "name": "ChatGPT-User",
      "operator": "OpenAI",
      "agentClass": "user_action_agent",
      "userAgentTokens": [
        "ChatGPT-User"
      ],
      "robotsTokens": [
        "ChatGPT-User"
      ],
      "respectsRobots": "partial",
      "verification": {
        "ipRangeUrls": [
          "https://openai.com/chatgpt-user.json"
        ]
      },
      "purpose": "Visits pages when ChatGPT or Custom GPT users ask questions or use GPT Actions.",
      "docsUrl": "https://developers.openai.com/api/docs/bots",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "openai-chatgpt-agent",
      "name": "ChatGPT agent",
      "operator": "OpenAI",
      "agentClass": "agentic_browser",
      "userAgentTokens": [],
      "robotsTokens": [],
      "respectsRobots": "unknown",
      "verification": {
        "signatureAgents": [
          "https://chatgpt.com"
        ]
      },
      "purpose": "ChatGPT's browsing agent; identified by RFC 9421 signatures, not by User-Agent.",
      "docsUrl": "https://help.openai.com/en/articles/11845367-chatgpt-agent-allowlisting",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "anthropic-claude-searchbot",
      "name": "Claude-SearchBot",
      "operator": "Anthropic",
      "agentClass": "ai_search_crawler",
      "userAgentTokens": [
        "Claude-SearchBot"
      ],
      "robotsTokens": [
        "Claude-SearchBot"
      ],
      "respectsRobots": "yes",
      "verification": {
        "ipRangeUrls": [
          "https://claude.com/crawling/bots.json"
        ]
      },
      "purpose": "Crawls the web to improve the quality of Claude's search results.",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "anthropic-claude-user",
      "name": "Claude-User",
      "operator": "Anthropic",
      "agentClass": "user_action_agent",
      "userAgentTokens": [
        "Claude-User"
      ],
      "robotsTokens": [
        "Claude-User"
      ],
      "respectsRobots": "yes",
      "verification": {
        "ipRangeUrls": [
          "https://claude.com/crawling/bots.json"
        ]
      },
      "purpose": "Fetches pages when a Claude user asks a question.",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "anthropic-claudebot",
      "name": "ClaudeBot",
      "operator": "Anthropic",
      "agentClass": "training_crawler",
      "userAgentTokens": [
        "ClaudeBot"
      ],
      "robotsTokens": [
        "ClaudeBot"
      ],
      "respectsRobots": "yes",
      "verification": {
        "ipRangeUrls": [
          "https://claude.com/crawling/bots.json"
        ]
      },
      "purpose": "Collects web content that could contribute to training Anthropic's models.",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "perplexity-user",
      "name": "Perplexity-User",
      "operator": "Perplexity",
      "agentClass": "user_action_agent",
      "userAgentTokens": [
        "Perplexity-User"
      ],
      "robotsTokens": [
        "Perplexity-User"
      ],
      "respectsRobots": "no",
      "verification": {
        "ipRangeUrls": [
          "https://www.perplexity.com/perplexity-user.json"
        ]
      },
      "purpose": "Fetches pages when a Perplexity user asks a question; generally ignores robots.txt.",
      "docsUrl": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "perplexity-perplexitybot",
      "name": "PerplexityBot",
      "operator": "Perplexity",
      "agentClass": "ai_search_crawler",
      "userAgentTokens": [
        "PerplexityBot"
      ],
      "robotsTokens": [
        "PerplexityBot"
      ],
      "respectsRobots": "yes",
      "verification": {
        "ipRangeUrls": [
          "https://www.perplexity.com/perplexitybot.json"
        ]
      },
      "purpose": "Surfaces and links websites in Perplexity search results.",
      "docsUrl": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "google-agent",
      "name": "Google-Agent",
      "operator": "Google",
      "agentClass": "agentic_browser",
      "userAgentTokens": [
        "Google-Agent"
      ],
      "robotsTokens": [],
      "respectsRobots": "unknown",
      "verification": {
        "ipRangeUrls": [
          "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json"
        ],
        "signatureAgents": [
          "https://agent.bot.goog"
        ]
      },
      "purpose": "Agents hosted on Google infrastructure that navigate the web and act on user request.",
      "docsUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "google-extended",
      "name": "Google-Extended",
      "operator": "Google",
      "agentClass": "training_crawler",
      "userAgentTokens": [],
      "robotsTokens": [
        "Google-Extended"
      ],
      "robotsOnly": true,
      "respectsRobots": "yes",
      "verification": {},
      "purpose": "robots.txt token controlling use of content for Gemini training and grounding; no separate crawler.",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "google-googlebot",
      "name": "Googlebot",
      "operator": "Google",
      "agentClass": "search_engine",
      "userAgentTokens": [
        "Googlebot"
      ],
      "robotsTokens": [
        "Googlebot"
      ],
      "respectsRobots": "yes",
      "verification": {
        "ipRangeUrls": [
          "https://developers.google.com/static/crawling/ipranges/common-crawlers.json"
        ],
        "reverseDnsSuffixes": [
          ".googlebot.com",
          ".google.com"
        ]
      },
      "purpose": "Google Search crawler; its index also feeds Google's AI features.",
      "docsUrl": "https://developers.google.com/crawling/ipranges/",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "apple-applebot-extended",
      "name": "Applebot-Extended",
      "operator": "Apple",
      "agentClass": "training_crawler",
      "userAgentTokens": [],
      "robotsTokens": [
        "Applebot-Extended"
      ],
      "robotsOnly": true,
      "respectsRobots": "yes",
      "verification": {},
      "purpose": "robots.txt token controlling use of crawled data to train Apple's foundation models; does not crawl.",
      "docsUrl": "https://support.apple.com/en-us/119829",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "apple-applebot",
      "name": "Applebot",
      "operator": "Apple",
      "agentClass": "search_engine",
      "userAgentTokens": [
        "Applebot"
      ],
      "robotsTokens": [
        "Applebot"
      ],
      "respectsRobots": "yes",
      "verification": {
        "ipRangeUrls": [
          "https://search.developer.apple.com/applebot.json"
        ],
        "reverseDnsSuffixes": [
          ".applebot.apple.com"
        ]
      },
      "purpose": "Apple's crawler for Siri, Spotlight and Safari search features.",
      "docsUrl": "https://support.apple.com/en-us/119829",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "microsoft-bingbot",
      "name": "Bingbot",
      "operator": "Microsoft",
      "agentClass": "search_engine",
      "userAgentTokens": [
        "bingbot"
      ],
      "robotsTokens": [
        "bingbot"
      ],
      "respectsRobots": "yes",
      "verification": {
        "ipRangeUrls": [
          "https://www.bing.com/toolbox/bingbot.json"
        ],
        "reverseDnsSuffixes": [
          ".search.msn.com"
        ]
      },
      "purpose": "Bing search crawler; its index also grounds Copilot answers.",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "meta-externalfetcher",
      "name": "Meta-ExternalFetcher",
      "operator": "Meta",
      "agentClass": "user_action_agent",
      "userAgentTokens": [
        "meta-externalfetcher"
      ],
      "robotsTokens": [
        "meta-externalfetcher"
      ],
      "respectsRobots": "no",
      "verification": {},
      "purpose": "Fetches individual links at a user's request, including for agentic AI tasks.",
      "docsUrl": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "meta-webindexer",
      "name": "Meta-WebIndexer",
      "operator": "Meta",
      "agentClass": "ai_search_crawler",
      "userAgentTokens": [
        "meta-webindexer"
      ],
      "robotsTokens": [
        "meta-webindexer"
      ],
      "respectsRobots": "yes",
      "verification": {},
      "purpose": "Analyzes content to improve Meta AI search results and citations.",
      "docsUrl": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "meta-externalagent",
      "name": "Meta-ExternalAgent",
      "operator": "Meta",
      "agentClass": "training_crawler",
      "userAgentTokens": [
        "meta-externalagent"
      ],
      "robotsTokens": [
        "meta-externalagent"
      ],
      "respectsRobots": "yes",
      "verification": {},
      "purpose": "Crawls for training foundation AI models and product indexing.",
      "docsUrl": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "meta-facebookexternalhit",
      "name": "FacebookExternalHit",
      "operator": "Meta",
      "agentClass": "preview_fetcher",
      "userAgentTokens": [
        "facebookexternalhit"
      ],
      "robotsTokens": [
        "facebookexternalhit"
      ],
      "respectsRobots": "partial",
      "verification": {},
      "purpose": "Fetches shared links to build previews (title, description, thumbnail).",
      "docsUrl": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "commoncrawl-ccbot",
      "name": "CCBot",
      "operator": "Common Crawl",
      "agentClass": "training_crawler",
      "userAgentTokens": [
        "CCBot"
      ],
      "robotsTokens": [
        "CCBot"
      ],
      "respectsRobots": "unknown",
      "verification": {},
      "purpose": "Builds the Common Crawl open web archive, widely used as AI training data.",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "bytedance-bytespider",
      "name": "Bytespider",
      "operator": "ByteDance",
      "agentClass": "training_crawler",
      "userAgentTokens": [
        "Bytespider"
      ],
      "robotsTokens": [
        "Bytespider"
      ],
      "respectsRobots": "unknown",
      "verification": {},
      "purpose": "ByteDance crawler associated with AI model data collection.",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "amazon-amazonbot",
      "name": "Amazonbot",
      "operator": "Amazon",
      "agentClass": "ai_search_crawler",
      "userAgentTokens": [
        "Amazonbot"
      ],
      "robotsTokens": [
        "Amazonbot"
      ],
      "respectsRobots": "unknown",
      "verification": {},
      "purpose": "Amazon crawler used for Alexa answers and other Amazon services.",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "huawei-petalbot",
      "name": "PetalBot",
      "operator": "Huawei",
      "agentClass": "ai_search_crawler",
      "userAgentTokens": [
        "PetalBot"
      ],
      "robotsTokens": [
        "PetalBot"
      ],
      "respectsRobots": "unknown",
      "verification": {},
      "purpose": "Crawler for Huawei's Petal Search.",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "duckduckgo-duckassistbot",
      "name": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "agentClass": "user_action_agent",
      "userAgentTokens": [
        "DuckAssistBot"
      ],
      "robotsTokens": [
        "DuckAssistBot"
      ],
      "respectsRobots": "unknown",
      "verification": {},
      "purpose": "Fetches pages to answer questions in DuckDuckGo's AI-assisted answers.",
      "checkedAt": "2026-11-04"
    },
    {
      "id": "mistral-user",
      "name": "MistralAI-User",
      "operator": "Mistral AI",
      "agentClass": "user_action_agent",
      "userAgentTokens": [
        "MistralAI-User"
      ],
      "robotsTokens": [
        "MistralAI-User"
      ],
      "respectsRobots": "unknown",
      "verification": {},
      "purpose": "Fetches pages when Le Chat users ask questions.",
      "checkedAt": "2026-11-04"
    }
  ]
}