{"slug": "common-crawl", "name": "Common Crawl", "category": "other", "category_name": "Other software", "description": "", "recommenders": 5, "commenters": 6, "warners": 1, "score": 3.627, "rising": -0.31, "first": "2026-02-14", "latest": "2026-08-05", "why": "You could probably extract a lot from https://commoncrawl.org/", "why_is_quote": true, "has_verdict": false, "sentiment": {"recommend": 5, "neutral": 0, "compare": 0, "warn": 1}, "spark": [0, 0, 0, 1, 2, 0, 0, 0, 1, 1, 0, 0], "listed": true, "loved": [], "caveat": "", "short_quote": "You could probably extract a lot from https://commoncrawl.org/", "kind": "", "icon": true, "domain": "commoncrawl.org", "github_repo": null, "official_url": "https://commoncrawl.org", "pricing": "", "aliases": ["Common Crawl"], "verdict": null, "praise": [{"label": "", "summary": "", "quotes": [{"id": 787027952, "item_id": 49189247, "url": "https://news.ycombinator.com/item?id=49189247", "by": "toomuchtodo", "date": "2026-08-05", "version": null, "stance": "recommend", "quote": "Anyone can leverage common crawl and/or run their own crawler", "excerpt": "Anyone can leverage common crawl and/or run their own crawler, to do so isn\u2019t novel. https://commoncrawl.org/", "context": "", "use_case": "", "switched_from": "", "story": "The next chapter of our AI momentum"}, {"id": 782452288, "item_id": 48903268, "url": "https://news.ycombinator.com/item?id=48903268", "by": "BrunoBernardino", "date": "2026-07-14", "version": null, "stance": "recommend", "quote": "Common Crawl is a really cool project!", "excerpt": "Common Crawl is a really cool project! We're looking to implement a similar EU-based version: https://openwebindex.eu", "context": "", "use_case": "", "switched_from": "", "story": "Ask HN: What Are You Working On? (July 2026)"}, {"id": 760644992, "item_id": 47540312, "url": "https://news.ycombinator.com/item?id=47540312", "by": "onion2k", "date": "2026-03-27", "version": null, "stance": "recommend", "quote": "You could probably extract a lot from https://commoncrawl.org/", "excerpt": "You could probably extract a lot from https://commoncrawl.org/", "context": "", "use_case": "", "switched_from": "", "story": "21,864 Yugoslavian .yu Domains"}, {"id": 754125936, "item_id": 47132871, "url": "https://news.ycombinator.com/item?id=47132871", "by": "lovelearning", "date": "2026-02-23", "version": null, "stance": "recommend", "quote": "look into CommonCrawl first", "excerpt": "As an experiment, it's interesting. If anyone actually needs such a dataset, look into CommonCrawl first. I feel using something that already exists will be more cooperative and considerate than everyone overloading every website with their spider. https://commoncrawl.org/overview", "context": "", "use_case": "", "switched_from": "", "story": "Crawling a billion web pages in just over 24 hours, in 2025"}]}], "caveats": [{"label": "", "summary": "", "quotes": [{"id": 752327264, "item_id": 47020454, "url": "https://news.ycombinator.com/item?id=47020454", "by": "username223", "date": "2026-02-14", "version": null, "stance": "warn", "quote": "Common Crawl is a deeply dishonest front for AI data scraping.", "excerpt": "I'm part of that small but (hopefully) growing percentage, because Common Crawl is a deeply dishonest front for AI data scraping. Quoting Wikipedia: \"\"\" In November 2025, an investigation by technology journalist Alex Reisner for The Atlantic revealed that Common Crawl lied when it claimed it respected paywalls in its scraping and requests from publishers to have their content removed from its databases. It included ", "context": "", "use_case": "", "switched_from": "", "story": "Internet Increasingly Becoming Unarchivable"}]}], "who": [], "tags": [], "use_cases": ["web crawl dataset"], "compared": [], "makers": [], "threads": [{"url": "https://news.ycombinator.com/item?id=47490871", "title": "21,864 Yugoslavian .yu Domains", "mentions": 2}, {"url": "https://news.ycombinator.com/item?id=49184755", "title": "The next chapter of our AI momentum", "mentions": 1}, {"url": "https://news.ycombinator.com/item?id=48884984", "title": "Ask HN: What Are You Working On? (July 2026)", "mentions": 1}, {"url": "https://news.ycombinator.com/item?id=47117886", "title": "Crawling a billion web pages in just over 24 hours, in 2025", "mentions": 1}, {"url": "https://news.ycombinator.com/item?id=47017138", "title": "Internet Increasingly Becoming Unarchivable", "mentions": 1}], "sentiment_by_month": {"2025-11": {}, "2025-12": {}, "2026-01": {}, "2026-02": {"warn": 1, "recommend": 1}, "2026-03": {"recommend": 2}, "2026-04": {}, "2026-05": {}, "2026-06": {}, "2026-07": {"recommend": 1}, "2026-08": {"recommend": 1}, "2026-09": {}, "2026-10": {}}, "versions": [], "version_unnamed": 0, "family_ref": null, "runs_on": null, "lines": [], "quotes": [{"id": 787027952, "item_id": 49189247, "url": "https://news.ycombinator.com/item?id=49189247", "by": "toomuchtodo", "date": "2026-08-05", "version": null, "stance": "recommend", "quote": "Anyone can leverage common crawl and/or run their own crawler", "excerpt": "Anyone can leverage common crawl and/or run their own crawler, to do so isn\u2019t novel. https://commoncrawl.org/", "context": "", "use_case": "", "switched_from": "", "story": "The next chapter of our AI momentum"}, {"id": 782452288, "item_id": 48903268, "url": "https://news.ycombinator.com/item?id=48903268", "by": "BrunoBernardino", "date": "2026-07-14", "version": null, "stance": "recommend", "quote": "Common Crawl is a really cool project!", "excerpt": "Common Crawl is a really cool project! We're looking to implement a similar EU-based version: https://openwebindex.eu", "context": "", "use_case": "", "switched_from": "", "story": "Ask HN: What Are You Working On? (July 2026)"}, {"id": 760737552, "item_id": 47546097, "url": "https://news.ycombinator.com/item?id=47546097", "by": "ks2048", "date": "2026-03-27", "version": null, "stance": "recommend", "quote": "", "excerpt": "Here's the latest (although, it looks truncated at those having > 1M pages), https://commoncrawl.github.io/cc-crawl-statistics/plots/tld/latestcrawl.html", "context": "", "use_case": "Web crawl dataset", "switched_from": "", "story": "21,864 Yugoslavian .yu Domains"}, {"id": 760644992, "item_id": 47540312, "url": "https://news.ycombinator.com/item?id=47540312", "by": "onion2k", "date": "2026-03-27", "version": null, "stance": "recommend", "quote": "You could probably extract a lot from https://commoncrawl.org/", "excerpt": "You could probably extract a lot from https://commoncrawl.org/", "context": "", "use_case": "", "switched_from": "", "story": "21,864 Yugoslavian .yu Domains"}, {"id": 754125936, "item_id": 47132871, "url": "https://news.ycombinator.com/item?id=47132871", "by": "lovelearning", "date": "2026-02-23", "version": null, "stance": "recommend", "quote": "look into CommonCrawl first", "excerpt": "As an experiment, it's interesting. If anyone actually needs such a dataset, look into CommonCrawl first. I feel using something that already exists will be more cooperative and considerate than everyone overloading every website with their spider. https://commoncrawl.org/overview", "context": "", "use_case": "", "switched_from": "", "story": "Crawling a billion web pages in just over 24 hours, in 2025"}, {"id": 752327264, "item_id": 47020454, "url": "https://news.ycombinator.com/item?id=47020454", "by": "username223", "date": "2026-02-14", "version": null, "stance": "warn", "quote": "Common Crawl is a deeply dishonest front for AI data scraping.", "excerpt": "I'm part of that small but (hopefully) growing percentage, because Common Crawl is a deeply dishonest front for AI data scraping. Quoting Wikipedia: \"\"\" In November 2025, an investigation by technology journalist Alex Reisner for The Atlantic revealed that Common Crawl lied when it claimed it respected paywalls in its scraping and requests from publishers to have their content removed from its databases. It included ", "context": "", "use_case": "", "switched_from": "", "story": "Internet Increasingly Becoming Unarchivable"}]}