{
    "ts": "2026-08-30T16:18:41+00:00",
    "next_run_in_s": 0,
    "sources": [
        {
            "key": "google-blog",
            "name": "Google Search Central blog",
            "category": "official-blog",
            "url": "https://developers.google.com/search/blog/feed.xml",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "google-ai-features",
            "name": "Google: AI features and your website",
            "category": "official-doc",
            "url": "https://developers.google.com/search/docs/appearance/ai-features",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "google-robots",
            "name": "Google: robots.txt documentation",
            "category": "official-doc",
            "url": "https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "openai-bots",
            "name": "OpenAI: Overview of crawlers",
            "category": "official-doc",
            "url": "https://platform.openai.com/docs/bots",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "openai-gptbot-ip",
            "name": "OpenAI GPTBot IP ranges (gptbot.json)",
            "category": "registry",
            "url": "https://openai.com/gptbot.json",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "openai-searchbot-ip",
            "name": "OpenAI OAI-SearchBot IP ranges",
            "category": "registry",
            "url": "https://openai.com/searchbot.json",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "openai-user-ip",
            "name": "OpenAI ChatGPT-User IP ranges",
            "category": "registry",
            "url": "https://openai.com/chatgpt-user.json",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "anthropic-news",
            "name": "Anthropic news",
            "category": "official-blog",
            "url": "https://www.anthropic.com/news",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "amazon-amazonbot",
            "name": "Amazon: Amazonbot documentation",
            "category": "official-doc",
            "url": "https://developer.amazon.com/amazonbot",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "meta-crawlers",
            "name": "Meta: external crawlers documentation",
            "category": "official-doc",
            "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "apple-applebot",
            "name": "Apple: About Applebot",
            "category": "official-doc",
            "url": "https://support.apple.com/en-us/119829",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "ddg-bot",
            "name": "DuckDuckGo: DuckBot info",
            "category": "official-doc",
            "url": "https://duckduckgo.com/duckduckbot",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "bing-webmasters",
            "name": "Bing Webmaster resources",
            "category": "official-doc",
            "url": "https://www.bing.com/webmasters/about",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "commoncrawl-news",
            "name": "Common Crawl news",
            "category": "official-blog",
            "url": "https://commoncrawl.org/blog/",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "llms-txt-spec",
            "name": "llms.txt proposal specification",
            "category": "spec",
            "url": "https://raw.githubusercontent.com/answerdotai/llms-txt/main/README.md",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "schema-releases",
            "name": "schema.org releases",
            "category": "spec",
            "url": "https://schema.org/docs/releases.html",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "darkvisitors",
            "name": "Dark Visitors crawler registry",
            "category": "registry",
            "url": "https://darkvisitors.com/agents",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "seroundtable",
            "name": "Search Engine Roundtable feed",
            "category": "industry",
            "url": "https://www.seroundtable.com/index.rdf",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        },
        {
            "key": "sel-feed",
            "name": "Search Engine Land feed",
            "category": "industry",
            "url": "https://searchengineland.com/feed",
            "status": "pending",
            "http": null,
            "bytes": null,
            "hash12": null,
            "last_seen": null,
            "last_changed": null
        }
    ]
}