{"generatedAt":"2026-10-19T19:03:29.069Z","ipSources":[{"key":"openai-gptbot","operator":"OpenAI","url":"https://openai.com/gptbot.json"},{"key":"openai-searchbot","operator":"OpenAI","url":"https://openai.com/searchbot.json"},{"key":"openai-chatgpt-user","operator":"OpenAI","url":"https://openai.com/chatgpt-user.json"},{"key":"openai-adsbot","operator":"OpenAI","url":"https://openai.com/adsbot.json"},{"key":"anthropic","operator":"Anthropic","url":"https://claude.com/crawling/bots.json","note":"One flat list for ClaudeBot, Claude-User and Claude-SearchBot."},{"key":"perplexity-bot","operator":"Perplexity","url":"https://www.perplexity.com/perplexitybot.json"},{"key":"perplexity-user","operator":"Perplexity","url":"https://www.perplexity.com/perplexity-user.json"},{"key":"google-common","operator":"Google","url":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json"},{"key":"google-agents","operator":"Google","url":"https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json"},{"key":"google-fetchers","operator":"Google","url":"https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json"},{"key":"google-googlebot","operator":"Google","url":"https://developers.google.com/static/search/apis/ipranges/googlebot.json"},{"key":"apple","operator":"Apple","url":"http://search.developer.apple.com/applebot.json"},{"key":"commoncrawl","operator":"Common Crawl","url":"https://index.commoncrawl.org/ccbot.json"},{"key":"mistral-index","operator":"Mistral AI","url":"https://mistral.ai/mistralai-index-ips.json"},{"key":"mistral-user","operator":"Mistral AI","url":"https://mistral.ai/mistralai-user-ips.json"},{"key":"duckduckgo","operator":"DuckDuckGo","url":"https://duckduckgo.com/duckassistbot.json"}],"agents":[{"slug":"gptbot","name":"GPTBot","operator":"OpenAI","category":"ai-training-crawler","tokens":["GPTBot"],"robots":"yes","source":"curated","ipSource":"openai-gptbot","docs":"https://developers.openai.com/api/docs/bots","description":"OpenAI's training crawler."},{"slug":"oai-searchbot","name":"OAI-SearchBot","operator":"OpenAI","category":"ai-search-index","tokens":["OAI-SearchBot"],"robots":"yes","source":"curated","ipSource":"openai-searchbot","docs":"https://developers.openai.com/api/docs/bots","description":"Builds the index behind ChatGPT search results."},{"slug":"chatgpt-user","name":"ChatGPT-User","operator":"OpenAI","category":"ai-user-fetch","tokens":["ChatGPT-User"],"robots":"partial","source":"curated","ipSource":"openai-chatgpt-user","docs":"https://developers.openai.com/api/docs/bots","description":"Fetches a page because a ChatGPT user asked about it. OpenAI says robots.txt rules may not apply."},{"slug":"oai-adsbot","name":"OAI-AdsBot","operator":"OpenAI","category":"ai-tooling","tokens":["OAI-AdsBot"],"robots":"unknown","source":"curated","ipSource":"openai-adsbot","docs":"https://developers.openai.com/api/docs/bots","description":"Checks landing pages for ads shown in ChatGPT."},{"slug":"chatgpt-agent","name":"ChatGPT agent","operator":"OpenAI","category":"ai-browsing-agent","tokens":["ChatGPT Agent","ChatGPT-Agent"],"robots":"no","source":"curated","description":"ChatGPT's browsing agent. Signs requests with Web Bot Auth (Signature-Agent: chatgpt.com)."},{"slug":"claudebot","name":"ClaudeBot","operator":"Anthropic","category":"ai-training-crawler","tokens":["ClaudeBot"],"robots":"yes","source":"curated","ipSource":"anthropic","docs":"https://support.claude.com/en/articles/8896518","description":"Anthropic's training crawler."},{"slug":"claude-user","name":"Claude-User","operator":"Anthropic","category":"ai-user-fetch","tokens":["Claude-User"],"robots":"yes","source":"curated","ipSource":"anthropic","docs":"https://support.claude.com/en/articles/8896518","description":"Fetches a page because a Claude user asked about it."},{"slug":"claude-searchbot","name":"Claude-SearchBot","operator":"Anthropic","category":"ai-search-index","tokens":["Claude-SearchBot"],"robots":"yes","source":"curated","ipSource":"anthropic","docs":"https://support.claude.com/en/articles/8896518","description":"Improves Claude's search results."},{"slug":"claude-code","name":"Claude Code","operator":"Anthropic","category":"ai-coding-agent","tokens":["Claude-Code"],"robots":"unknown","source":"curated","description":"Coding agent fetching pages during a task."},{"slug":"anthropic-legacy","name":"anthropic-ai / Claude-Web (legacy)","operator":"Anthropic","category":"ai-training-crawler","tokens":["anthropic-ai","Claude-Web"],"robots":"unknown","source":"curated","controlTokenOnly":true,"description":"Legacy robots.txt names; not seen as live user agents."},{"slug":"perplexitybot","name":"PerplexityBot","operator":"Perplexity","category":"ai-search-index","tokens":["PerplexityBot"],"robots":"yes","source":"curated","ipSource":"perplexity-bot","docs":"https://docs.perplexity.ai/guides/bots","description":"Indexes pages for Perplexity answers. Perplexity states it is not used for training."},{"slug":"perplexity-user","name":"Perplexity-User","operator":"Perplexity","category":"ai-user-fetch","tokens":["Perplexity-User"],"robots":"no","source":"curated","ipSource":"perplexity-user","docs":"https://docs.perplexity.ai/guides/bots","description":"Fetches a page on a user's behalf. Perplexity says it generally ignores robots.txt."},{"slug":"google-extended","name":"Google-Extended","operator":"Google","category":"ai-training-crawler","tokens":["Google-Extended"],"robots":"yes","source":"curated","controlTokenOnly":true,"docs":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","description":"robots.txt control token for Gemini training and grounding. Never appears as a live user agent."},{"slug":"googleother","name":"GoogleOther","operator":"Google","category":"ai-tooling","tokens":[],"robots":"yes","source":"curated","prefixTokens":["GoogleOther"],"ipSource":"google-common","docs":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","description":"Generic crawler used by Google product teams for research and development, including AI."},{"slug":"google-cloudvertexbot","name":"Google-CloudVertexBot","operator":"Google","category":"ai-tooling","tokens":["Google-CloudVertexBot"],"robots":"yes","source":"curated","ipSource":"google-common","docs":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","description":"Crawls on behalf of Vertex AI Agent Builder customers."},{"slug":"google-agent","name":"Google-Agent","operator":"Google","category":"ai-browsing-agent","tokens":["Google-Agent"],"robots":"no","source":"curated","ipSource":"google-agents","docs":"https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers","description":"Google's browsing agent (added March 2026). Ignores robots.txt by design."},{"slug":"google-geminino","name":"Google-GeminiNotebook","operator":"Google","category":"ai-user-fetch","tokens":["Google-GeminiNotebook","Google-NotebookLM"],"robots":"no","source":"curated","ipSource":"google-fetchers","docs":"https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers","description":"Fetches sources a NotebookLM / Gemini user added."},{"slug":"googleagent-legacy","name":"GoogleAgent-Mariner / URLContext","operator":"Google","category":"ai-browsing-agent","tokens":["GoogleAgent-Mariner","GoogleAgent-URLContext"],"robots":"no","source":"curated","description":"Earlier agent tokens; likely superseded by Google-Agent."},{"slug":"gemini-deep-research","name":"Gemini Deep Research","operator":"Google","category":"ai-user-fetch","tokens":["Gemini-Deep-Research"],"robots":"no","source":"curated","description":"Fetches pages during a Gemini Deep Research run."},{"slug":"google-gemini-cli","name":"Gemini CLI","operator":"Google","category":"ai-coding-agent","tokens":["Google-Gemini-CLI"],"robots":"unknown","source":"curated"},{"slug":"googlebot","name":"Googlebot","operator":"Google","category":"search-engine","tokens":["Googlebot"],"robots":"yes","source":"curated","ipSource":"google-googlebot","description":"Classic web search crawler. Not counted as AI."},{"slug":"applebot","name":"Applebot","operator":"Apple","category":"search-engine","tokens":["Applebot"],"robots":"yes","source":"curated","ipSource":"apple","rdns":[".applebot.apple.com"],"docs":"https://support.apple.com/en-us/119829","description":"Powers Siri and Spotlight search. Apple may also use it for training unless Applebot-Extended is disallowed."},{"slug":"applebot-extended","name":"Applebot-Extended","operator":"Apple","category":"ai-training-crawler","tokens":["Applebot-Extended"],"robots":"yes","source":"curated","controlTokenOnly":true,"description":"robots.txt control token for Apple model training. Never appears as a live user agent."},{"slug":"meta-externalagent","name":"Meta-ExternalAgent","operator":"Meta","category":"ai-training-crawler","tokens":["meta-externalagent"],"robots":"yes","source":"curated","docs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","description":"Training and indexing crawler. Meta publishes no IP ranges, so it cannot be verified."},{"slug":"meta-externalfetcher","name":"Meta-ExternalFetcher","operator":"Meta","category":"ai-user-fetch","tokens":["meta-externalfetcher"],"robots":"partial","source":"curated","docs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","description":"User-triggered fetch for Meta AI. May bypass robots.txt."},{"slug":"meta-webindexer","name":"Meta-WebIndexer","operator":"Meta","category":"ai-search-index","tokens":["meta-webindexer"],"robots":"yes","source":"curated","docs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","description":"Index behind Meta AI search."},{"slug":"facebookbot","name":"FacebookBot","operator":"Meta","category":"ai-training-crawler","tokens":["FacebookBot"],"robots":"yes","source":"curated","description":"Older Meta crawler used to improve language models."},{"slug":"amazonbot","name":"Amazonbot","operator":"Amazon","category":"ai-training-crawler","tokens":["Amazonbot"],"robots":"yes","source":"curated","docs":"https://developer.amazon.com/amazonbot","description":"Improves Alexa and trains Amazon models. IP list is published as an HTML page."},{"slug":"amzn-searchbot","name":"Amzn-SearchBot","operator":"Amazon","category":"ai-search-index","tokens":["Amzn-SearchBot"],"robots":"yes","source":"curated","docs":"https://developer.amazon.com/amazonbot"},{"slug":"amzn-user","name":"Amzn-User","operator":"Amazon","category":"ai-user-fetch","tokens":["Amzn-User"],"robots":"yes","source":"curated","docs":"https://developer.amazon.com/amazonbot","description":"Live fetch on behalf of an Alexa user."},{"slug":"novaact","name":"Nova Act","operator":"Amazon","category":"ai-browsing-agent","tokens":["NovaAct"],"robots":"unknown","source":"curated"},{"slug":"amazon-buy-for-me","name":"Buy For Me","operator":"Amazon","category":"ai-browsing-agent","tokens":["AmazonBuyForMe"],"robots":"unknown","source":"curated"},{"slug":"mistral-training","name":"MistralAI-Training","operator":"Mistral AI","category":"ai-training-crawler","tokens":["MistralAI-Training"],"robots":"yes","source":"curated","docs":"https://docs.mistral.ai/robots"},{"slug":"mistral-index","name":"MistralAI-Index","operator":"Mistral AI","category":"ai-search-index","tokens":["MistralAI-Index"],"robots":"yes","source":"curated","ipSource":"mistral-index","docs":"https://docs.mistral.ai/robots"},{"slug":"mistral-user","name":"MistralAI-User","operator":"Mistral AI","category":"ai-user-fetch","tokens":["MistralAI-User"],"robots":"yes","source":"curated","ipSource":"mistral-user","docs":"https://docs.mistral.ai/robots"},{"slug":"ccbot","name":"CCBot","operator":"Common Crawl","category":"ai-training-crawler","tokens":["CCBot"],"robots":"yes","source":"curated","ipSource":"commoncrawl","rdns":[".crawl.commoncrawl.org"],"docs":"https://commoncrawl.org/ccbot","description":"Open crawl dataset that feeds many model training runs."},{"slug":"duckassistbot","name":"DuckAssistBot","operator":"DuckDuckGo","category":"ai-user-fetch","tokens":["DuckAssistBot"],"robots":"yes","source":"curated","ipSource":"duckduckgo"},{"slug":"duckduckbot","name":"DuckDuckBot","operator":"DuckDuckGo","category":"search-engine","tokens":["DuckDuckBot"],"robots":"yes","source":"curated"},{"slug":"bingbot","name":"Bingbot","operator":"Microsoft","category":"search-engine","tokens":["bingbot"],"robots":"yes","source":"curated","rdns":[".search.msn.com"],"description":"Bing's crawler; the same index grounds Copilot answers. Not counted as AI."},{"slug":"bytespider","name":"Bytespider","operator":"ByteDance","category":"ai-training-crawler","tokens":["Bytespider"],"robots":"no","source":"curated","description":"Widely reported to ignore robots.txt."},{"slug":"tiktokspider","name":"TikTokSpider","operator":"ByteDance","category":"ai-training-crawler","tokens":["TikTokSpider"],"robots":"unknown","source":"curated"},{"slug":"deepseekbot","name":"DeepSeekBot","operator":"DeepSeek","category":"ai-training-crawler","tokens":["DeepSeekBot"],"robots":"unknown","source":"curated"},{"slug":"kimibot","name":"KimiBot","operator":"Moonshot AI","category":"ai-training-crawler","tokens":["KimiBot"],"robots":"unknown","source":"curated"},{"slug":"kimi-user","name":"Kimi-User","operator":"Moonshot AI","category":"ai-user-fetch","tokens":["Kimi-User"],"robots":"unknown","source":"curated"},{"slug":"kimi-searchbot","name":"Kimi-SearchBot","operator":"Moonshot AI","category":"ai-search-index","tokens":["Kimi-SearchBot"],"robots":"unknown","source":"curated"},{"slug":"tongyibot","name":"TongyiBot","operator":"Alibaba","category":"ai-training-crawler","tokens":["TongyiBot"],"robots":"unknown","source":"curated"},{"slug":"yiyanbot","name":"YiyanBot","operator":"Baidu","category":"ai-training-crawler","tokens":["YiyanBot"],"robots":"unknown","source":"curated"},{"slug":"pangubot","name":"PanguBot","operator":"Huawei","category":"ai-training-crawler","tokens":["PanguBot"],"robots":"unknown","source":"curated"},{"slug":"petalbot","name":"PetalBot","operator":"Huawei","category":"ai-search-index","tokens":["PetalBot"],"robots":"yes","source":"curated"},{"slug":"yandex-additional","name":"YandexAdditional","operator":"Yandex","category":"ai-training-crawler","tokens":["YandexAdditional","YandexAdditionalBot"],"robots":"unknown","source":"curated","description":"Feeds YandexGPT."},{"slug":"yandexbot","name":"YandexBot","operator":"Yandex","category":"search-engine","tokens":["YandexBot"],"robots":"yes","source":"curated"},{"slug":"baiduspider","name":"Baiduspider","operator":"Baidu","category":"search-engine","tokens":["Baiduspider"],"robots":"yes","source":"curated"},{"slug":"cohere","name":"cohere-ai","operator":"Cohere","category":"ai-training-crawler","tokens":["cohere-ai","cohere-training-data-crawler"],"robots":"unknown","source":"curated"},{"slug":"ai2bot","name":"AI2Bot","operator":"Allen Institute for AI","category":"ai-training-crawler","tokens":["AI2Bot","Ai2Bot-Dolma"],"robots":"yes","source":"curated"},{"slug":"timpibot","name":"Timpibot","operator":"Timpi","category":"ai-search-index","tokens":["Timpibot"],"robots":"unknown","source":"curated"},{"slug":"youbot","name":"YouBot","operator":"You.com","category":"ai-search-index","tokens":["YouBot"],"robots":"yes","source":"curated"},{"slug":"omgili","name":"omgili / webz.io","operator":"Webz.io","category":"ai-training-crawler","tokens":["omgili","omgilibot"],"robots":"unknown","source":"curated","description":"Sells crawled data, including to model trainers."},{"slug":"manus","name":"Manus","operator":"Butterfly Effect","category":"ai-browsing-agent","tokens":["Manus-User"],"robots":"unknown","source":"curated"},{"slug":"devin","name":"Devin","operator":"Cognition","category":"ai-browsing-agent","tokens":["Devin"],"robots":"unknown","source":"curated"},{"slug":"twinagent","name":"TwinAgent","operator":"Twin","category":"ai-browsing-agent","tokens":["TwinAgent"],"robots":"unknown","source":"curated"},{"slug":"lightpanda","name":"Lightpanda","operator":"Lightpanda","category":"ai-browsing-agent","tokens":["Lightpanda"],"robots":"unknown","source":"curated","description":"Headless browser built for AI agents."},{"slug":"cursor","name":"Cursor","operator":"Anysphere","category":"ai-coding-agent","tokens":["Cursor"],"robots":"unknown","source":"curated"},{"slug":"opencode","name":"opencode","operator":"SST","category":"ai-coding-agent","tokens":["opencode"],"robots":"unknown","source":"curated"},{"slug":"trae","name":"Trae","operator":"ByteDance","category":"ai-coding-agent","tokens":["Trae"],"robots":"unknown","source":"curated"},{"slug":"firecrawl","name":"Firecrawl","operator":"Firecrawl","category":"ai-tooling","tokens":["FirecrawlAgent"],"robots":"unknown","source":"curated"},{"slug":"exa","name":"Exa","operator":"Exa","category":"ai-search-index","tokens":["ExaBot","ExaSearchBot"],"robots":"unknown","source":"curated"},{"slug":"tavily","name":"Tavily","operator":"Tavily","category":"ai-tooling","tokens":["TavilyBot"],"robots":"unknown","source":"curated"},{"slug":"crawl4ai","name":"Crawl4AI","operator":"Crawl4AI","category":"ai-tooling","tokens":["Crawl4AI"],"robots":"unknown","source":"curated"},{"slug":"diffbot","name":"Diffbot","operator":"Diffbot","category":"ai-tooling","tokens":["Diffbot"],"robots":"partial","source":"curated"},{"slug":"diffbot-user","name":"Diffbot-User","operator":"Diffbot","category":"ai-user-fetch","tokens":["Diffbot-User"],"robots":"unknown","source":"curated"},{"slug":"brightbot","name":"Brightbot","operator":"Bright Data","category":"ai-tooling","tokens":["Brightbot"],"robots":"unknown","source":"curated"},{"slug":"apifybot","name":"ApifyBot","operator":"Apify","category":"ai-tooling","tokens":["ApifyBot"],"robots":"unknown","source":"curated"},{"slug":"imagesiftbot","name":"ImagesiftBot","operator":"Hive","category":"ai-tooling","tokens":["ImagesiftBot"],"robots":"unknown","source":"curated"},{"slug":"img2dataset","name":"img2dataset","operator":"various","category":"ai-training-crawler","tokens":["img2dataset"],"robots":"unknown","source":"curated"},{"slug":"slurp","name":"Yahoo Slurp","operator":"Yahoo","category":"search-engine","tokens":["Slurp"],"robots":"yes","source":"curated"},{"slug":"link-previewers","name":"Link previewers","operator":"various","category":"other-bot","tokens":["facebookexternalhit","Twitterbot","LinkedInBot","Slackbot","Discordbot","TelegramBot","WhatsApp","Embedly"],"robots":"unknown","source":"curated"},{"slug":"seo-crawlers","name":"SEO crawlers","operator":"various","category":"other-bot","tokens":["AhrefsBot","SemrushBot","MJ12bot","DotBot","DataForSeoBot","BLEXBot"],"robots":"unknown","source":"curated"},{"slug":"monitors","name":"Uptime monitors & platform bots","operator":"various","category":"other-bot","tokens":["UptimeRobot","Pingdom","vercel-screenshot","Vercelbot","Lighthouse","Chrome-Lighthouse","PageSpeed"],"robots":"unknown","source":"curated"},{"slug":"addsearchbot","name":"AddSearchBot","operator":"Unknown operator","category":"ai-search-index","tokens":["AddSearchBot"],"robots":"unknown","description":"AddSearchBot is a web crawler that indexes website content for AddSearch's AI-powered site search solution, collecting data to provide fast and accurate search results. More info can be found at https://knownagents.com/agents/addsearchbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"agenttimes","name":"AgentTimes","operator":"[The Agent Times](https://theagenttimes.com/about)","category":"ai-training-crawler","tokens":["AgentTimes"],"robots":"unknown","description":"Scrapes data for AI news aggregation and republishing.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"ai2bot-deepresearcheval","name":"AI2Bot-DeepResearchEval","operator":"Ai2, a non-profit AI research institute","category":"ai-user-fetch","tokens":["AI2Bot-DeepResearchEval"],"robots":"unknown","description":"Ai2Bot-DeepResearchEval is operated by Ai2, a non-profit AI research institute. It's used to collect and scan resources used in deep research queries performed by Ai2's o… More info can be found at https://knownagents.com/agents/ai2bot-deepresearcheval","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"aihitbot","name":"aiHitBot","operator":"[aiHit](https://www.aihitdata.com/about)","category":"ai-training-crawler","tokens":["aiHitBot"],"robots":"yes","description":"Scrapes data for AI systems.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"aiwebindex","name":"AIWebIndex","operator":"[Lyrenth](https://lyrenth.com)","category":"ai-search-index","tokens":["AIWebIndex"],"robots":"yes","description":"Builds an index of public pages and serves them to AI agents as extracted, readable text with attribution and a link back to the source. Does not train foundation models on crawled content. Identity can be checked three ways: published IP ranges at https://lyrenth.com/bot/ip-ranges.json, forward-confirmed reverse DNS under lyrenth.com, and Web Bot Auth signatures (RFC 9421). Full policy at https://lyrenth.com/crawler-policy","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"amazon-kendra","name":"amazon-kendra","operator":"Amazon","category":"ai-search-index","tokens":["amazon-kendra"],"robots":"yes","description":"Amazon Kendra is a highly accurate intelligent search service that enables your users to search unstructured data using natural language. It returns specific answers to questions, giving users an experience that's close to interacting with a human expert. It is highly scalable and capable of meeting performance demands, tightly integrated with other AWS services such as Amazon S3 and Amazon Lex, and offers enterprise-grade security.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"amazon-qbusiness","name":"amazon-QBusiness","operator":"Unknown operator","category":"ai-user-fetch","tokens":["amazon-QBusiness"],"robots":"unknown","description":"amazon-QBusiness is an Amazon Q Business web crawler that fetches and indexes web content for Amazon Q Business applications. More info can be found at https://knownagents.com/agents/amazon-qbusiness","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"andibot","name":"Andibot","operator":"[Andi](https://andisearch.com/)","category":"ai-user-fetch","tokens":["Andibot"],"robots":"unknown","description":"Scrapes website and provides AI summary.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"anomura","name":"Anomura","operator":"[Direqt](https://direqt.ai)","category":"ai-search-index","tokens":["Anomura"],"robots":"yes","description":"Anomura is Direqt's search crawler, it discovers and indexes pages their customers websites.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"apifywebsitecontentcrawler","name":"ApifyWebsiteContentCrawler","operator":"Unknown operator","category":"ai-tooling","tokens":["ApifyWebsiteContentCrawler"],"robots":"unknown","description":"ApifyWebsiteContentCrawler is a web crawler by Apify that extracts and downloads full website content for use in AI, data analysis, and automation workflows. More info can be found at https://knownagents.com/agents/apifywebsitecontentcrawler","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"aranet-searchbot","name":"Aranet-SearchBot","operator":"Unknown operator","category":"ai-browsing-agent","tokens":["Aranet-SearchBot"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/aranet-searchbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"atlassian-bot","name":"atlassian-bot","operator":"[Atlassian](https://www.atlassian.com)","category":"ai-user-fetch","tokens":["atlassian-bot"],"robots":"yes","description":"atlassian-bot is a web crawler used to index website content for its AI search, assistants and agents available in its Rovo GenAI product.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"awario","name":"Awario","operator":"Awario","category":"ai-training-crawler","tokens":["Awario"],"robots":"unknown","description":"Awario is an AI data scraper operated by Awario. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://knownagents.com/agents/awario","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"azureai-searchbot","name":"AzureAI-SearchBot","operator":"Unknown operator","category":"ai-search-index","tokens":["AzureAI-SearchBot"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/azureai-searchbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"bedrockbot","name":"bedrockbot","operator":"[Amazon](https://amazon.com)","category":"ai-training-crawler","tokens":["bedrockbot"],"robots":"yes","description":"Connects to and crawls URLs that have been selected for use in a user's AWS bedrock application.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"bigsur-ai","name":"bigsur.ai","operator":"Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for busi…","category":"ai-user-fetch","tokens":["bigsur.ai"],"robots":"unknown","description":"bigsur.ai is a web crawler operated by Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for busi… More info can be found at https://knownagents.com/agents/bigsur-ai","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"bravebot","name":"Bravebot","operator":"https://safe.search.brave.com/help/brave-search-crawler","category":"ai-tooling","tokens":["Bravebot"],"robots":"yes","description":"Bravebot is a web crawler by Brave that indexes pages for Brave Search, providing search data and AI-optimized context to power chatbots, agents, and RAG pipelines. More info can be found at https://knownagents.com/agents/bravebot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"buddybot","name":"BuddyBot","operator":"[BuddyBotLearning](https://www.buddybotlearning.com)","category":"ai-tooling","tokens":["BuddyBot"],"robots":"unknown","description":"BuddyBot is a voice-controlled AI learning companion targeted at childhooded STEM education.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"channel3bot","name":"Channel3Bot","operator":"Unknown operator","category":"ai-search-index","tokens":["Channel3Bot"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/channel3bot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"chatglm-spider","name":"ChatGLM-Spider","operator":"Unknown operator","category":"ai-training-crawler","tokens":["ChatGLM-Spider"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/chatglm-spider","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"cloudflare-autorag","name":"Cloudflare-AutoRAG","operator":"[Cloudflare](https://developers.cloudflare.com/autorag)","category":"ai-search-index","tokens":["Cloudflare-AutoRAG"],"robots":"yes","description":"AutoRAG is an all-in-one AI search solution.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"cloudvertexbot","name":"CloudVertexBot","operator":"Unknown operator","category":"ai-training-crawler","tokens":["CloudVertexBot"],"robots":"unknown","description":"CloudVertexBot is a Google-operated crawler available to site owners to request targeted crawls of their own sites for AI training purposes on the Vertex AI platform. More info can be found at https://knownagents.com/agents/cloudvertexbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"cotoyogi","name":"Cotoyogi","operator":"[ROIS](https://ds.rois.ac.jp/en_center8/en_crawler/)","category":"ai-training-crawler","tokens":["Cotoyogi"],"robots":"yes","description":"Scrapes data for AI training in Japanese language.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"cragcrawler","name":"CragCrawler","operator":"CragSoftware, a Brazil-based software company specializing in data engineering and AI web scraping services","category":"ai-tooling","tokens":["CragCrawler"],"robots":"unknown","description":"CragCrawler is a web scraping bot operated by CragSoftware, a Brazil-based software company specializing in data engineering and AI web scraping services. The bot is used… More info can be found at https://knownagents.com/agents/cragcrawler","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"crawlspace","name":"Crawlspace","operator":"[Crawlspace](https://crawlspace.dev)","category":"ai-tooling","tokens":["Crawlspace"],"robots":"yes","description":"Crawlspace is a web crawler platform that fetches and extracts website content for AI agents, RAG applications, and structured data workflows. More info can be found at https://knownagents.com/agents/crawlspace","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"datenbank-crawler","name":"Datenbank Crawler","operator":"Datenbank","category":"ai-training-crawler","tokens":["Datenbank Crawler"],"robots":"unknown","description":"Datenbank Crawler is an AI data scraper operated by Datenbank. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://knownagents.com/agents/datenbank-crawler","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"echobot-bot","name":"Echobot Bot","operator":"Echobox","category":"ai-training-crawler","tokens":["Echobot Bot"],"robots":"unknown","description":"Echobot Bot is an AI data scraper operated by Echobox. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://knownagents.com/agents/echobot-bot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"echoboxbot","name":"EchoboxBot","operator":"[Echobox](https://echobox.com)","category":"ai-tooling","tokens":["EchoboxBot"],"robots":"unknown","description":"Supports company's AI-powered social and email management products.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"factset-spyderbot","name":"Factset_spyderbot","operator":"[Factset](https://www.factset.com/ai)","category":"ai-training-crawler","tokens":["Factset_spyderbot"],"robots":"unknown","description":"Scrapes data for AI training.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"friendlycrawler","name":"FriendlyCrawler","operator":"Unknown","category":"ai-training-crawler","tokens":["FriendlyCrawler"],"robots":"yes","description":"Unclear who the operator is; but data is used for training/machine learning.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"geisthaus-pagefetcher","name":"GeistHaus-PageFetcher","operator":"GeistHaus, a company developing AI systems for therapy and psychological assessment","category":"ai-user-fetch","tokens":["GeistHaus-PageFetcher"],"robots":"unknown","description":"GeistHaus-PageFetcher is a web crawler operated by GeistHaus, a company developing AI systems for therapy and psychological assessment. This bot fetches web pages as part… More info can be found at https://knownagents.com/agents/geisthaus-pagefetcher","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"google-firebase","name":"Google-Firebase","operator":"Google","category":"ai-tooling","tokens":["Google-Firebase"],"robots":"unknown","description":"Supports Google's Firebase AI products.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"henkbot","name":"HenkBot","operator":"Unknown operator","category":"ai-tooling","tokens":["HenkBot"],"robots":"unknown","description":"Henkbot crawls the web on behalf of Valyu, an AI search infrastructure provider that indexes content for use in AI-powered retrieval pipelines. More info can be found at https://knownagents.com/agents/henkbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"iaskbot","name":"iAskBot","operator":"Unknown operator","category":"ai-browsing-agent","tokens":["iAskBot"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/iaskbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"iaskspider","name":"iaskspider","operator":"Unknown operator","category":"ai-browsing-agent","tokens":["iaskspider"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/iaskspider","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"icc-crawler","name":"ICC-Crawler","operator":"[NICT](https://nict.go.jp)","category":"ai-training-crawler","tokens":["ICC-Crawler"],"robots":"yes","description":"Use the collected data for artificial intelligence technologies; provide data to third parties, including commercial companies; those companies can use the data for their own business.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"imagespider","name":"imageSpider","operator":"Unknown operator","category":"ai-training-crawler","tokens":["imageSpider"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/imagespider","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"isscyberriskcrawler","name":"ISSCyberRiskCrawler","operator":"[ISS-Corporate](https://iss-cyber.com)","category":"ai-training-crawler","tokens":["ISSCyberRiskCrawler"],"robots":"no","description":"Used to train machine learning based models to quantify cyber risk.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"kagi-fetcher","name":"kagi-fetcher","operator":"Kagi that fetches web content to answer user queries through Kagi AI, their suite of AI-powered tools including Assistant, Res…","category":"ai-user-fetch","tokens":["kagi-fetcher"],"robots":"unknown","description":"kagi-fetcher is an AI Assistant operated by Kagi that fetches web content to answer user queries through Kagi AI, their suite of AI-powered tools including Assistant, Res… More info can be found at https://knownagents.com/agents/kagi-fetcher","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"kangaroo-bot","name":"Kangaroo Bot","operator":"Unknown operator","category":"ai-training-crawler","tokens":["Kangaroo Bot"],"robots":"unknown","description":"Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://knownagents.com/agents/kangaroo-bot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"klaviyoaibot","name":"KlaviyoAIBot","operator":"[Klaviyo](https://www.klaviyo.com)","category":"ai-user-fetch","tokens":["KlaviyoAIBot"],"robots":"yes","description":"KlaviyoAIBot is Klaviyo's web crawler that fetches publicly available pages from domains explicitly connected to user accounts to power the Kai Customer Agent feature. Th… More info can be found at https://knownagents.com/agents/klaviyoaibot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"kunatocrawler","name":"KunatoCrawler","operator":"Unknown operator","category":"ai-browsing-agent","tokens":["KunatoCrawler"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/kunatocrawler","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"laion-huggingface-processor","name":"laion-huggingface-processor","operator":"Unknown operator","category":"ai-training-crawler","tokens":["laion-huggingface-processor"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/laion-huggingface-processor","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"laiondownloader","name":"LAIONDownloader","operator":"[Large-scale Artificial Intelligence Open Network](https://laion.ai/)","category":"ai-search-index","tokens":["LAIONDownloader"],"robots":"no","description":"LAIONDownloader is a bot by LAION, a non-profit organization that provides datasets, tools and models to liberate machine learning research.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"linerbot","name":"LinerBot","operator":"Unknown operator","category":"ai-user-fetch","tokens":["LinerBot"],"robots":"unknown","description":"LinerBot is the web crawler used by Liner AI assistant to gather information from academic sources and websites to provide accurate answers with line-by-line source citat… More info can be found at https://knownagents.com/agents/linerbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"linguee-bot","name":"Linguee Bot","operator":"[Linguee](https://www.linguee.com)","category":"ai-tooling","tokens":["Linguee Bot"],"robots":"no","description":"Linguee Bot is a web crawler used by Linguee to gather training data for its AI powered translation service.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"linkupbot","name":"LinkupBot","operator":"Unknown operator","category":"ai-search-index","tokens":["LinkupBot"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/linkupbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"mozilla-tabstack","name":"Mozilla-Tabstack","operator":"Mozilla that performs programmatic, AI-driven interactions with web content through Tabstack","category":"ai-tooling","tokens":["Mozilla-Tabstack"],"robots":"yes","description":"Mozilla-Tabstack is an AI agent operated by Mozilla that performs programmatic, AI-driven interactions with web content through Tabstack. More info can be found at https://knownagents.com/agents/mozilla-tabstack","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"mycentralaiscraperbot","name":"MyCentralAIScraperBot","operator":"Unknown operator","category":"ai-training-crawler","tokens":["MyCentralAIScraperBot"],"robots":"unknown","description":"Operator and data use is unclear at this time.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"nagetbot","name":"NagetBot","operator":"Naget Inc (founded by Chris Samarinas, headquarter in Amherst, Massachusetts)","category":"ai-training-crawler","tokens":["NagetBot"],"robots":"unknown","description":"'Naget revolutionizes content discovery through an AI-powered ecosystem that transforms how we generate, organize, share, and discover valuable content.' (https://naget.com/) User-agent string links https://naget.ai/bot which yields 404.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"netestate-imprint-crawler","name":"netEstate Imprint Crawler","operator":"netEstate","category":"ai-training-crawler","tokens":["netEstate Imprint Crawler"],"robots":"unknown","description":"netEstate Imprint Crawler is an AI data scraper operated by netEstate. If you think this is incorrect or can provide additional detail about its purpose, please contact us. More info can be found at https://knownagents.com/agents/netestate-imprint-crawler","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"newsai","name":"newsai","operator":"Unknown operator","category":"ai-training-crawler","tokens":["newsai"],"robots":"unknown","description":"User-agent string doen't contain an URL and there multiple sites using the newsai brand.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"notebooklm","name":"NotebookLM","operator":"Unknown operator","category":"ai-user-fetch","tokens":["NotebookLM"],"robots":"unknown","description":"NotebookLM is an AI-powered research and note-taking assistant that helps users synthesize information from their own uploaded sources, such as documents, transcripts, or web content. It can generate summaries, answer questions, and highlight key themes from the materials you provide, acting like a personalized research companion built on Google's Gemini model. NotebookLM fetches source URLs when users add them to their notebooks, enabling the AI to access and analyze those pages for context and insights. More info can be found at https://knownagents.com/agents/google-notebooklm","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"openai","name":"OpenAI","operator":"[OpenAI](https://openai.com)","category":"ai-tooling","tokens":["OpenAI"],"robots":"yes","description":"The purpose of this bot is unclear at this time but it is a member of OpenAI's suite of crawlers.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"panscient","name":"Panscient","operator":"[Panscient](https://panscient.com)","category":"ai-training-crawler","tokens":["Panscient"],"robots":"yes","description":"Compiles data on businesses and business professionals that is structured using AI and machine learning.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"phindbot","name":"PhindBot","operator":"[phind](https://www.phind.com/)","category":"ai-user-fetch","tokens":["PhindBot"],"robots":"unknown","description":"Phind is an AI-powered answer engine designed for developers, offering technical answers and code examples. It uses real-time web search and specialized AI models to prov… More info can be found at https://knownagents.com/agents/phindbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"poggio-citations","name":"Poggio-Citations","operator":"Poggio, a company that provides AI sales enablement tools for creating tailored narratives, business cases, and account plan…","category":"ai-user-fetch","tokens":["Poggio-Citations"],"robots":"unknown","description":"Poggio-Citations is a web crawler operated by Poggio, a company that provides AI sales enablement tools for creating tailored narratives, business cases, and account plan… More info can be found at https://knownagents.com/agents/poggio-citations","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"poseidon-research-crawler","name":"Poseidon Research Crawler","operator":"[Poseidon Research](https://www.poseidonresearch.com)","category":"ai-search-index","tokens":["Poseidon Research Crawler"],"robots":"unknown","description":"Lab focused on scaling the interpretability research necessary to make better AI systems possible.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"qualifiedbot","name":"QualifiedBot","operator":"[Qualified](https://www.qualified.com)","category":"ai-user-fetch","tokens":["QualifiedBot"],"robots":"unknown","description":"QualifiedBot is Qualified's web crawler that analyzes customer websites to provide contextual information for their AI-powered chatbots and conversational marketing platf… More info can be found at https://knownagents.com/agents/qualifiedbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"querit-searchbot","name":"Querit-SearchBot","operator":"Querit that indexes web content for their search API service, which is designed to provide real-time search results for larg…","category":"ai-tooling","tokens":["Querit-SearchBot"],"robots":"unknown","description":"Querit-SearchBot is a web crawler operated by Querit that indexes web content for their search API service, which is designed to provide real-time search results for larg… More info can be found at https://knownagents.com/agents/querit-searchbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"queritbot","name":"QueritBot","operator":"Querit, a company providing a search API for large language model integration","category":"ai-tooling","tokens":["QueritBot"],"robots":"unknown","description":"QueritBot is a web crawler operated by Querit, a company providing a search API for large language model integration. This bot indexes web content to power the real-time … More info can be found at https://knownagents.com/agents/queritbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"quillbot","name":"QuillBot","operator":"[Quillbot](https://quillbot.com)","category":"ai-tooling","tokens":["QuillBot"],"robots":"unknown","description":"Operated by QuillBot as part of their suite of AI product offerings.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"reflectionbot","name":"Reflectionbot","operator":"[Reflection](https://reflection.ai/)","category":"ai-browsing-agent","tokens":["Reflectionbot"],"robots":"unknown","description":"An undocumented crawler whose user agent links to Reflection, a company that builds AI models.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"sbintuitionsbot","name":"SBIntuitionsBot","operator":"[SB Intuitions](https://www.sbintuitions.co.jp/en/)","category":"ai-tooling","tokens":["SBIntuitionsBot"],"robots":"yes","description":"AI development and information analysis","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"semrushbot-ocob","name":"SemrushBot-OCOB","operator":"[Semrush](https://www.semrush.com/)","category":"ai-tooling","tokens":["SemrushBot-OCOB"],"robots":"yes","description":"Data collected is used for the ContentShake AI tool reports.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"semrushbot-swa","name":"SemrushBot-SWA","operator":"[Semrush](https://www.semrush.com/)","category":"ai-user-fetch","tokens":["SemrushBot-SWA"],"robots":"yes","description":"Data collected is used for the SEO Writing Assistant tool to check if URL is accessible.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"shap-user","name":"Shap-User","operator":"Unknown operator","category":"ai-user-fetch","tokens":["Shap-User"],"robots":"unknown","description":"Shap-User accesses web content on behalf of users of Parallel Web Systems products. It identifies user-initiated requests rather than automatic web crawling. More info can be found at https://knownagents.com/agents/shap-user","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"shapbot","name":"ShapBot","operator":"[Parallel](https://parallel.ai)","category":"ai-tooling","tokens":["ShapBot"],"robots":"yes","description":"ShapBot is a web crawler by Parallel that collects and structures web content to power its search, extraction, and deep research APIs, providing AI agents with high-accur… More info can be found at https://knownagents.com/agents/shapbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"sidetrade-indexer-bot","name":"Sidetrade indexer bot","operator":"[Sidetrade](https://www.sidetrade.com)","category":"ai-training-crawler","tokens":["Sidetrade indexer bot"],"robots":"unknown","description":"AI product training.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"terra-cotta","name":"Terra Cotta","operator":"Unknown operator","category":"ai-tooling","tokens":["Terra Cotta"],"robots":"unknown","description":"Terra Cotta is Ceramic's web crawler that indexes public content to power their web-scale search API for AI and LLMs. More info can be found at https://knownagents.com/agents/terra-cotta","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"terracotta","name":"TerraCotta","operator":"[Ceramic AI](https://ceramic.ai/)","category":"ai-tooling","tokens":["TerraCotta"],"robots":"yes","description":"TerraCotta is Ceramic's web crawler that indexes public content to power their web-scale search API for AI and LLMs. More info can be found at https://knownagents.com/agents/terracotta","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"thinkbot","name":"Thinkbot","operator":"[Thinkbot](https://www.thinkbot.agency)","category":"ai-tooling","tokens":["Thinkbot"],"robots":"no","description":"Collects data for analysis on AI usage and automation.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"useai","name":"UseAI","operator":"Unknown operator","category":"ai-user-fetch","tokens":["UseAI"],"robots":"unknown","description":"UseAI is a web crawler associated with Use AI, a platform that provides an AI workspace where users can chat with AI models, research the web, and perform various tasks. … More info can be found at https://knownagents.com/agents/useai","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"velenpublicwebcrawler","name":"VelenPublicWebCrawler","operator":"[Velen Crawler](https://velen.io)","category":"ai-training-crawler","tokens":["VelenPublicWebCrawler"],"robots":"yes","description":"\"Our goal with this crawler is to build business datasets and machine learning models to better understand the web.\"","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"wardbot","name":"WARDBot","operator":"WEBSPARK","category":"ai-training-crawler","tokens":["WARDBot"],"robots":"unknown","description":"WARDBot is an AI data scraper operated by WEBSPARK. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://knownagents.com/agents/wardbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"webzio-extended","name":"Webzio-Extended","operator":"Unknown operator","category":"ai-training-crawler","tokens":["Webzio-Extended"],"robots":"unknown","description":"Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://knownagents.com/agents/webzio-extended","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"wpbot","name":"wpbot","operator":"[QuantumCloud](https://www.quantumcloud.com)","category":"ai-tooling","tokens":["wpbot"],"robots":"unknown","description":"wpbot is a used to support the functionality of the AI Chatbot for WordPress plugin. It supports the use of customer models, data collection and customer support.","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"wrtnbot","name":"WRTNBot","operator":"Unknown operator","category":"ai-browsing-agent","tokens":["WRTNBot"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/wrtnbot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"},{"slug":"zanistabot","name":"ZanistaBot","operator":"Unknown operator","category":"ai-search-index","tokens":["ZanistaBot"],"robots":"unknown","description":"Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/zanistabot","docs":"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json","source":"ai-robots-txt"}]}