{"slug": "swe-rebench", "name": "SWE-rebench", "category": "ai", "category_name": "AI tools and apps", "description": "", "recommenders": 3, "commenters": 3, "warners": 0, "score": 2.613, "rising": -0.98, "first": "2026-03-05", "latest": "2026-04-11", "why": "This team is doing a good job.  They use problems that were created in last 30days to avoid training set leakage.", "why_is_quote": true, "has_verdict": false, "sentiment": {"recommend": 3, "neutral": 0, "compare": 0, "warn": 0}, "spark": [0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0], "listed": true, "loved": [], "caveat": "", "short_quote": "This team is doing a good job.  They use problems that were created in last 30days to avoid training set leakage.", "kind": "", "icon": true, "domain": "swe-rebench.com", "github_repo": null, "official_url": "https://swe-rebench.com", "pricing": "", "aliases": ["SWE-rebench"], "verdict": null, "praise": [{"label": "", "summary": "", "quotes": [{"id": 763744432, "item_id": 47734027, "url": "https://news.ycombinator.com/item?id=47734027", "by": "bbcc90", "date": "2026-04-11", "version": null, "stance": "recommend", "quote": "This team is doing a good job.  They use problems that were created in last 30days to avoid training set leakage.", "excerpt": "Yes good evals are really hard - that\u2019s not really news. This team is doing a good job. They use problems that were created in last 30days to avoid training set leakage. https://swe-rebench.com/", "context": "", "use_case": "Coding agent benchmarking", "switched_from": "", "story": "How We Broke Top AI Agent Benchmarks: And What Comes Next"}, {"id": 761905185, "item_id": 47619074, "url": "https://news.ycombinator.com/item?id=47619074", "by": "nabakin", "date": "2026-04-02", "version": null, "stance": "recommend", "quote": "SWE Rebench seems to have decent protection against training data leaks", "excerpt": "I know Arc AGI 2 has a private test set and they have a good amount of results[0] but it's not a conventional benchmark. Looking around, SWE Rebench seems to have decent protection against training data leaks[1]. Kagi has one that is fully private[2]. One on HuggingFace that claims to be fully private[3]. SimpleBench[4]. HLE has a private test set apparently[5]. LiveBench[6]. Scale has some private benchmarks but not", "context": "", "use_case": "", "switched_from": "", "story": "Google releases Gemma 4 open models"}, {"id": 756344912, "item_id": 47271557, "url": "https://news.ycombinator.com/item?id=47271557", "by": "kaufmann", "date": "2026-03-05", "version": null, "stance": "recommend", "quote": "I would recommend https://swe-rebench.com for comparison.", "excerpt": "I would recommend https://swe-rebench.com for comparison. It is always based on new problems.", "context": "", "use_case": "", "switched_from": "", "story": "GPT-5.4 Thinking System Card"}]}], "caveats": [], "who": [], "tags": [], "use_cases": ["coding agent benchmarking"], "compared": [], "makers": [], "threads": [{"url": "https://news.ycombinator.com/item?id=47733217", "title": "How We Broke Top AI Agent Benchmarks: And What Comes Next", "mentions": 1}, {"url": "https://news.ycombinator.com/item?id=47616361", "title": "Google releases Gemma 4 open models", "mentions": 1}, {"url": "https://news.ycombinator.com/item?id=47265045", "title": "GPT-5.4 Thinking System Card", "mentions": 1}], "sentiment_by_month": {"2025-11": {}, "2025-12": {}, "2026-01": {}, "2026-02": {}, "2026-03": {"recommend": 1}, "2026-04": {"recommend": 2}, "2026-05": {}, "2026-06": {}, "2026-07": {}, "2026-08": {}, "2026-09": {}, "2026-10": {}}, "versions": [], "version_unnamed": 0, "family_ref": null, "runs_on": null, "lines": [], "quotes": [{"id": 763744432, "item_id": 47734027, "url": "https://news.ycombinator.com/item?id=47734027", "by": "bbcc90", "date": "2026-04-11", "version": null, "stance": "recommend", "quote": "This team is doing a good job.  They use problems that were created in last 30days to avoid training set leakage.", "excerpt": "Yes good evals are really hard - that\u2019s not really news. This team is doing a good job. They use problems that were created in last 30days to avoid training set leakage. https://swe-rebench.com/", "context": "", "use_case": "Coding agent benchmarking", "switched_from": "", "story": "How We Broke Top AI Agent Benchmarks: And What Comes Next"}, {"id": 761905185, "item_id": 47619074, "url": "https://news.ycombinator.com/item?id=47619074", "by": "nabakin", "date": "2026-04-02", "version": null, "stance": "recommend", "quote": "SWE Rebench seems to have decent protection against training data leaks", "excerpt": "I know Arc AGI 2 has a private test set and they have a good amount of results[0] but it's not a conventional benchmark. Looking around, SWE Rebench seems to have decent protection against training data leaks[1]. Kagi has one that is fully private[2]. One on HuggingFace that claims to be fully private[3]. SimpleBench[4]. HLE has a private test set apparently[5]. LiveBench[6]. Scale has some private benchmarks but not", "context": "", "use_case": "", "switched_from": "", "story": "Google releases Gemma 4 open models"}, {"id": 756344912, "item_id": 47271557, "url": "https://news.ycombinator.com/item?id=47271557", "by": "kaufmann", "date": "2026-03-05", "version": null, "stance": "recommend", "quote": "I would recommend https://swe-rebench.com for comparison.", "excerpt": "I would recommend https://swe-rebench.com for comparison. It is always based on new problems.", "context": "", "use_case": "", "switched_from": "", "story": "GPT-5.4 Thinking System Card"}]}