{ "dataset": "swebench-verified-longitudinal", "schema_version": "yylo_seo_longitudinal_dataset.v1", "dataset_version": 1, "description": "Bounded longitudinal extract of the public SWE-bench Verified coding-agent leaderboard: one dated entry per evaluated system on one fixed, human-validated 500-instance task set, republished for citation and analysis with its limitations.", "publisher": "yylo.dev", "canonical_url": "https://yylo.dev/guides/longitudinal-coding-agent-benchmark-dataset", "download_url": "https://yylo.dev/data/swebench-verified-longitudinal-v1.json", "source": { "surface": "https://www.swebench.com/", "board": "Verified", "board_description": "A human-validated subset of 500 SWE-bench instances for reliable evaluation of coding agents and language models.", "payload": "embedded application/json leaderboard document in the fetched page", "fetched_at": "2026-08-29", "surface_last_modified": "2026-08-10", "source_document_sha256": "sha256:c9043b06f15c37becc1bcfa1d42b509a4439762f675eadab43e6bef40adace98" }, "population": { "entries": 180, "first_entry_date": "2023-10-10", "last_entry_date": "2026-02-26", "order": "as published: resolved_percent descending, ties in published order" }, "field_notes": { "entry": "leaderboard display name of the evaluated system, as published", "entry_date": "entry date on the leaderboard (ISO); the run precedes the listing", "source_folder": "the entry's folder identifier on the source surface; unique within this extract and stable mapping back to the source materials", "agent": "agent/scaffold name as published", "agent_org": "submitting organization as published; null when the source carries none", "model": "display name of the model (or ensemble label) as published", "model_org": "model organization as published; null when the source carries none", "models": "raw model identifiers extracted from the entry's `Model:` tags; more than one member means an ensemble or multi-model pipeline; empty when the source carries no model tag (the display name then carries the identity, e.g. Undisclosed)", "model_release_date": "model release date as published, normalized from YYYYMMDD to ISO; null when the source carries none", "open_source_system": "the source marks the agent system as open source", "open_source_model": "the source marks the model as open source", "resolved_percent": "percent of the 500 instances resolved, republished exactly as published", "attempts": "attempt policy tag extracted from the entry's `System:` tag: `1`, `2`, or `2+`", "verified": "the source's `checked` marker normalized to boolean; null when the source carries none", "verified_note": "verbatim source marker when the source states verification in note form (verified stays false)", "total_cost_usd": "total reported run cost in USD as published; null when not reported", "mean_cost_per_instance_usd": "mean cost per instance in USD as published; null when not reported", "mean_api_calls_per_instance": "mean API calls per instance as published; null when not reported", "agent_version": "mini-SWE-agent release for the board's frozen-scaffold runs; null for all other systems", "reasoning_effort": "declared reasoning effort when the source carries one; null otherwise", "system_url": "submitter's system page(s) when the source carries them — occasionally several for one entry; null otherwise" }, "rights_note": "The underlying entries belong to their submitters and the SWE-bench team; this bounded extract exists for citation and analysis with attribution to the source surface, and it is not a YYLO measurement.", "entries_sha256": "sha256:cbd2393a235c0276211fb4d7035c943ef7eb7e4be4902be99a5733f6380d02b6", "entries": [ { "entry": "live-SWE-agent + Claude 4.5 Opus medium (20251101)", "entry_date": "2025-12-15", "source_folder": "20251215_livesweagent_claude-opus-4-5", "agent": "live-SWE-agent", "agent_org": "UIUC", "model": "Claude 4.5 Opus", "model_org": "Anthropic", "models": [ "claude-opus-4-5-20251101" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 79.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": "medium", "system_url": "https://github.com/OpenAutoCoder/live-swe-agent" }, { "entry": "Sonar Foundation Agent + Claude 4.5 Opus", "entry_date": "2025-12-05", "source_folder": "20251205_sonar-foundation-agent_claude-opus-4-5", "agent": "Sonar Foundation Agent", "agent_org": "Sonar", "model": "Claude 4.5 Opus", "model_org": "Anthropic", "models": [ "claude-opus-4-5" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 79.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.sonarsource.com" }, { "entry": "TRAE + Doubao-Seed-Code", "entry_date": "2025-09-28", "source_folder": "20250928_trae_doubao_seed_code", "agent": "TRAE", "agent_org": "ByteDance", "model": "Doubao-Seed-Code", "model_org": "ByteDance", "models": [ "Doubao-Seed-Code", "Doubao-Seed-1.6" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 78.8, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": [ "https://github.com/bytedance/trae-agent", "https://seed.bytedance.com/" ] }, { "entry": "live-SWE-agent + Gemini 3 Pro Preview (2025-11-18)", "entry_date": "2025-11-20", "source_folder": "20251120_livesweagent_gemini-3-pro-preview", "agent": "live-SWE-agent", "agent_org": "UIUC", "model": "Gemini 3 Pro Preview", "model_org": "Google DeepMind", "models": [ "gemini-3-pro-preview" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 77.4, "attempts": "1", "verified": false, "verified_note": "false (See README.md for info on how to get your results verified)", "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/OpenAutoCoder/live-swe-agent" }, { "entry": "Atlassian Rovo Dev (2025-09-02)", "entry_date": "2025-09-02", "source_folder": "20250902_atlassian-rovo-dev", "agent": "Atlassian Rovo Dev", "agent_org": "Atlassian", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-sonnet-4-20250514", "gpt-5" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 76.8, "attempts": "2", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.atlassian.com/rovo-dev" }, { "entry": "EPAM AI/Run Developer Agent v20250719 + Claude 4 Sonnet", "entry_date": "2025-08-04", "source_folder": "20250804_epam-ai-run-claude-4-sonnet", "agent": "EPAM AI/Run Developer Agent", "agent_org": "EPAM Systems, Inc.", "model": "Claude 4 Sonnet", "model_org": "Anthropic", "models": [ "claude-sonnet-4-20250514" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 76.8, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.epam.com/services/artificial-intelligence" }, { "entry": "mini-SWE-agent + Claude 4.5 Opus (high)", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_claude-4-5-opus-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Claude 4.5 Opus", "model_org": "Anthropic", "models": [ "claude-4-5-opus" ], "model_release_date": "2025-11-24", "open_source_system": true, "open_source_model": false, "resolved_percent": 76.8, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 376.9539985, "mean_cost_per_instance_usd": 0.753907997, "mean_api_calls_per_instance": 32.896, "agent_version": "2.0.0", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "ACoder", "entry_date": "2025-08-19", "source_folder": "20250819_ACoder", "agent": "ACoder", "agent_org": "ACoder", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-4-sonnet", "claude-4.1-opus", "gpt-5-0807-global", "gemini-2.5-pro-06-17" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 76.4, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/ACoder-AI/ACoder" }, { "entry": "mini-SWE-agent + Gemini 3 Flash (high)", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_gemini-3-flash-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Gemini 3 Flash", "model_org": "Google DeepMind", "models": [ "gemini-3-flash-preview" ], "model_release_date": "2025-12-17", "open_source_system": true, "open_source_model": false, "resolved_percent": 75.8, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 177.98206539999998, "mean_cost_per_instance_usd": 0.35596413079999994, "mean_api_calls_per_instance": 56.126, "agent_version": "2.0.0", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + MiniMax M2.5 (high)", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_minimax-2-5-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "MiniMax M2.5", "model_org": "Minimax", "models": [ "minimax-m2.5" ], "model_release_date": "2026-02-12", "open_source_system": true, "open_source_model": true, "resolved_percent": 75.8, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 36.644993922, "mean_cost_per_instance_usd": 0.073289987844, "mean_api_calls_per_instance": 60.45, "agent_version": "2.0.0", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Warp", "entry_date": "2025-09-01", "source_folder": "20250901_warp", "agent": "Warp", "agent_org": "Warp", "model": "Multiple", "model_org": "OpenAI", "models": [ "gpt-5", "claude-sonnet-4-20250514" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 75.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.warp.dev" }, { "entry": "mini-SWE-agent + Claude 4.6 Opus", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_claude-4-6-opus", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Claude 4.6 Opus", "model_org": "Anthropic", "models": [ "claude-opus-4-6" ], "model_release_date": "2026-02-05", "open_source_system": true, "open_source_model": false, "resolved_percent": 75.6, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 275.76132225, "mean_cost_per_instance_usd": 0.5515226445, "mean_api_calls_per_instance": 28.932, "agent_version": "2.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "TRAE + Claude Sonnet 4 + Opus 4 + Sonnet 3.7 + Gemini 2.5 Pro", "entry_date": "2025-06-12", "source_folder": "20250612_trae", "agent": "TRAE", "agent_org": "TRAE", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-4-sonnet-20250522", "claude-4-opus-20250522", "claude-3-7-sonnet-20250219", "gemini-2.5-pro-preview-06-05" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 75.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/bytedance/trae-agent" }, { "entry": "Harness AI", "entry_date": "2025-07-31", "source_folder": "20250731_harness_ai", "agent": "Harness AI", "agent_org": "Harness", "model": "Claude Sonnet 4", "model_org": "Anthropic", "models": [ "claude-sonnet-4-20250514" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 74.8, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.harness.io/" }, { "entry": "Sonar Foundation Agent + Claude 4.5 Sonnet", "entry_date": "2025-11-03", "source_folder": "20251103_sonar-foundation-agent_claude-sonnet-4-5", "agent": "Sonar Foundation Agent", "agent_org": "Sonar", "model": "Claude 4.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-sonnet-4-5" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 74.8, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.sonarsource.com" }, { "entry": "Lingxi-v1.5_claude-4-sonnet-20250514", "entry_date": "2025-07-20", "source_folder": "20250720_Lingxi-v1.5_claude-4-sonnet-20250514", "agent": "Lingxi-v1.5", "agent_org": "Lingxi", "model": "Claude 4 Sonnet", "model_org": "Anthropic", "models": [ "claude-4-sonnet-20250514" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 74.6, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/nimasteryang/Lingxi" }, { "entry": "JoyCode + Claude 4 Sonnet + GPT-4.1", "entry_date": "2025-09-15", "source_folder": "20250915_JoyCode", "agent": "JoyCode", "agent_org": "JoyCode", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-4-sonnet", "gpt-4.1" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 74.6, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/jd-opensource/joycode-agent" }, { "entry": "Refact.ai Agent + Claude 4 Sonnet + o4-mini", "entry_date": "2025-06-03", "source_folder": "20250603_Refact_Agent_claude-4-sonnet", "agent": "Refact.ai Agent", "agent_org": "Refact.ai", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-4-sonnet", "o4-mini" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 74.4, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://refact.ai" }, { "entry": "Prometheus-v1.2.1 + GPT-5", "entry_date": "2025-10-15", "source_folder": "20251015_Prometheus_v1.2.1_gpt5", "agent": "Prometheus-v1.2.1", "agent_org": "EuniAI", "model": "GPT-5", "model_org": "OpenAI", "models": [ "gpt-5-2025-08-07" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 74.4, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://euni.ai/" }, { "entry": "mini-SWE-agent + Claude 4.5 Opus (20251101) (medium)", "entry_date": "2025-11-24", "source_folder": "20251124_mini-v1.16.0_claude-opus-4-5-20251101", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Claude 4.5 Opus", "model_org": "Anthropic", "models": [ "claude-opus-4-5-20251101" ], "model_release_date": "2025-11-24", "open_source_system": true, "open_source_model": false, "resolved_percent": 74.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 360.6224985, "mean_cost_per_instance_usd": 0.721244997, "mean_api_calls_per_instance": 38.042, "agent_version": "1.16.0", "reasoning_effort": "medium", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + Gemini 3 Pro Preview (2025-11-18)", "entry_date": "2025-11-18", "source_folder": "20251118_mini-v1.15.0_gemini-3-pro-preview-20251118", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Gemini 3 Pro Preview", "model_org": "Google DeepMind", "models": [ "gemini-3-pro-preview" ], "model_release_date": "2025-11-18", "open_source_system": true, "open_source_model": false, "resolved_percent": 74.2, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 229.978759475, "mean_cost_per_instance_usd": 0.45995751895000003, "mean_api_calls_per_instance": 40.326, "agent_version": "1.15.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Salesforce AI Research SAGE (OpenHands)", "entry_date": "2025-11-03", "source_folder": "20251103_SalesforceAIResearch_SAGE_OpenHands", "agent": "Salesforce AI Research SAGE", "agent_org": "Salesforce AI Research", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-sonnet-4.5", "gpt-5" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 73.8, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.salesforce.com/blog/sage-swe/" }, { "entry": "Tools + Claude 4 Opus (2025-05-22)", "entry_date": "2025-05-22", "source_folder": "20250522_tools_claude-4-opus", "agent": "Tools", "agent_org": null, "model": "Claude 4 Opus", "model_org": "Anthropic", "models": [ "claude-4-opus-20250514" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 73.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.anthropic.com/" }, { "entry": "Salesforce AI Research SAGE (bash-only)", "entry_date": "2025-10-21", "source_folder": "20251021_SalesforceAIResearch_SAGE_bash_only", "agent": "Salesforce AI Research SAGE", "agent_org": "Salesforce AI Research", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-sonnet-4.5", "gpt-5" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 73, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.salesforce.com/blog/sage-swe/" }, { "entry": "mini-SWE-agent + GPT 5.2 Codex", "entry_date": "2026-02-19", "source_folder": "20260219_mini-v2.0.0_gpt-5-2-codex", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 5.2 Codex", "model_org": "OpenAI", "models": [ "gpt-5-2-codex" ], "model_release_date": "2025-12-11", "open_source_system": true, "open_source_model": false, "resolved_percent": 72.8, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 224.71194935, "mean_cost_per_instance_usd": 0.4494238987, "mean_api_calls_per_instance": 28.072, "agent_version": "2.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + GLM 5 (high)", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_glm-5-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GLM 5", "model_org": "Z-AI", "models": [ "glm-5" ], "model_release_date": "2026-02-11", "open_source_system": true, "open_source_model": true, "resolved_percent": 72.8, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 267.19456811, "mean_cost_per_instance_usd": 0.53438913622, "mean_api_calls_per_instance": 76.176, "agent_version": "2.0.0", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + GPT 5.2 (high)", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_gpt-5-2-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 5.2", "model_org": "OpenAI", "models": [ "gpt-5-2" ], "model_release_date": "2025-12-11", "open_source_system": true, "open_source_model": false, "resolved_percent": 72.8, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 236.78350885, "mean_cost_per_instance_usd": 0.4735670177, "mean_api_calls_per_instance": 35.046, "agent_version": "2.0.0", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Tools + Claude 4 Sonnet (2025-05-22)", "entry_date": "2025-05-22", "source_folder": "20250522_tools_claude-4-sonnet", "agent": "Tools", "agent_org": null, "model": "Claude 4 Sonnet", "model_org": "Anthropic", "models": [ "claude-4-sonnet-20250514" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 72.4, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.anthropic.com/" }, { "entry": "OpenHands + GPT-5", "entry_date": "2025-08-07", "source_folder": "20250807_openhands_gpt5", "agent": "OpenHands", "agent_org": "OpenHands", "model": "GPT-5", "model_org": "OpenAI", "models": [ "openai/gpt-5-2025-08-07" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 71.8, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://all-hands.dev/" }, { "entry": "mini-SWE-agent + GPT 5.2 (2025-12-11) (high)", "entry_date": "2025-12-11", "source_folder": "20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 5.2", "model_org": "OpenAI", "models": [ "gpt-5.2-2025-12-11" ], "model_release_date": "2025-12-11", "open_source_system": true, "open_source_model": false, "resolved_percent": 71.8, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 260.1295018, "mean_cost_per_instance_usd": 0.5202590036, "mean_api_calls_per_instance": 19.762, "agent_version": "1.17.2", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + Claude 4.5 Sonnet (high)", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_claude-4-5-sonnet-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Claude 4.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-sonnet-4-5-20250929" ], "model_release_date": "2025-09-29", "open_source_system": true, "open_source_model": false, "resolved_percent": 71.4, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 328.9482492, "mean_cost_per_instance_usd": 0.6578964984000001, "mean_api_calls_per_instance": 48.3, "agent_version": "2.0.0", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Prometheus-v1.2 + GPT-5", "entry_date": "2025-09-29", "source_folder": "20250929_Prometheus_v1.2_gpt5", "agent": "Prometheus-v1.2", "agent_org": "EuniAI", "model": "GPT-5", "model_org": "OpenAI", "models": [ "gpt-5-2025-08-07" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 71.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://euni.ai/" }, { "entry": "Qodo Command", "entry_date": "2025-07-15", "source_folder": "20250715_qodo_command", "agent": "Qodo Command", "agent_org": "Qodo", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-sonnet-4-20250514", "gpt-4o" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 71.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.qodo.ai/" }, { "entry": "Bloop", "entry_date": "2025-07-10", "source_folder": "20250710_bloop", "agent": "Bloop", "agent_org": "Bloop AI", "model": "Claude 4 Sonnet", "model_org": "Anthropic", "models": [ "claude-4-sonnet-20250514" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 71.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://bloop.ai/" }, { "entry": "Lingxi v1.5 x Kimi K2", "entry_date": "2025-10-14", "source_folder": "20251014_Lingxi_kimi_k2", "agent": "Lingxi v1.5", "agent_org": "Huawei", "model": "Kimi K2", "model_org": "Moonshot AI", "models": [ "kimi-k2-0905-preview" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 71.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/lingxi-agent/Lingxi/tree/master" }, { "entry": "Warp", "entry_date": "2025-06-23", "source_folder": "20250623_warp", "agent": "Warp", "agent_org": "Warp", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-sonnet-4-20250514", "claude-3-7-sonnet-20250219", "gemini-2.5-pro", "gpt-4.1-2025-04-14" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 71, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.warp.dev" }, { "entry": "Moatless Tools + Claude 4 Sonnet", "entry_date": "2025-06-11", "source_folder": "20250611_moatless_claude-4-sonnet-20250514", "agent": "Moatless Tools", "agent_org": "Moatless AI", "model": "Claude 4 Sonnet", "model_org": "Anthropic", "models": [ "claude-4-sonnet-20250514" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 70.8, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/aorwall/moatless-tools" }, { "entry": "mini-SWE-agent + Kimi K2.5 (high)", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_kimi-k2-5-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Kimi K2.5", "model_org": "Moonshot AI", "models": [ "kimi-k2.5" ], "model_release_date": "2026-01-27", "open_source_system": true, "open_source_model": true, "resolved_percent": 70.8, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 73.276870372, "mean_cost_per_instance_usd": 0.14655374074400002, "mean_api_calls_per_instance": 51.178, "agent_version": "2.0.0", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "TRAE", "entry_date": "2025-05-19", "source_folder": "20250519_trae", "agent": "TRAE", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 70.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.trae.ai/" }, { "entry": "mini-SWE-agent + Claude 4.5 Sonnet (20250929)", "entry_date": "2025-09-29", "source_folder": "20250929_mini-v1.13.3_sonnet-4-5-20250929", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Claude 4.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-sonnet-4-5-20250929" ], "model_release_date": "2025-09-29", "open_source_system": true, "open_source_model": false, "resolved_percent": 70.6, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 279.16737045, "mean_cost_per_instance_usd": 0.5583347409, "mean_api_calls_per_instance": 50.988, "agent_version": "1.13.3", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Refact.ai Agent", "entry_date": "2025-05-15", "source_folder": "20250515_Refact_Agent", "agent": "Refact.ai Agent", "agent_org": "Refact.ai", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-3-7-sonnet-20250219", "o3", "o4-mini" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 70.4, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://refact.ai" }, { "entry": "OpenHands + Claude 4 Sonnet", "entry_date": "2025-05-24", "source_folder": "20250524_openhands_claude_4_sonnet", "agent": "OpenHands", "agent_org": "OpenHands", "model": "Claude 4 Sonnet", "model_org": "Anthropic", "models": [ "claude-4-sonnet-20250514" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 70.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://all-hands.dev/" }, { "entry": "Augment Agent v1", "entry_date": "2025-06-10", "source_folder": "20250610_augment_agent_v1", "agent": "Augment Agent v1", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 70.4, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.augmentcode.com" }, { "entry": "devlo", "entry_date": "2025-05-19", "source_folder": "20250519_devlo", "agent": "devlo", "agent_org": "devlo", "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-3-7-sonnet-20250219", "o3", "gemini-2.5-pro" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 70.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://devlo.ai/?utm_source=swe-bench" }, { "entry": "Zencoder (2025-04-30)", "entry_date": "2025-04-30", "source_folder": "20250430_zencoder_ai", "agent": "Zencoder", "agent_org": null, "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-3-7-sonnet-20250219", "o4-mini" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 70, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://zencoder.ai" }, { "entry": "mini-SWE-agent + DeepSeek V3.2 (high)", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_deepseek-3-2-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "DeepSeek V3.2", "model_org": "DeepSeek", "models": [ "deepseek-v3.2" ], "model_release_date": "2025-12-01", "open_source_system": true, "open_source_model": true, "resolved_percent": 70, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 223.92324825, "mean_cost_per_instance_usd": 0.4478464965, "mean_api_calls_per_instance": 88.506, "agent_version": "2.0.0", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "OpenHands + Qwen3-Coder-480B-A35B-Instruct", "entry_date": "2025-08-05", "source_folder": "20250805_openhands-Qwen3-Coder-480B-A35B-Instruct", "agent": "OpenHands", "agent_org": "Qwen", "model": "Qwen3-Coder-480B-A35B-Instruct", "model_org": "Qwen", "models": [ "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 69.6, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://qwenlm.github.io/blog/qwen3-coder/" }, { "entry": "mini-SWE-agent + Gemini 3 Pro", "entry_date": "2026-02-26", "source_folder": "20260226_mini-v2.0.0_gemini-3-pro-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Gemini 3 Pro", "model_org": "Google DeepMind", "models": [ "gemini-3-pro-preview" ], "model_release_date": "2025-11-18", "open_source_system": true, "open_source_model": false, "resolved_percent": 69.6, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 480.0099282, "mean_cost_per_instance_usd": 0.9600198564, "mean_api_calls_per_instance": 51.282, "agent_version": "2.0.0", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + GPT 5.2 (2025-12-11)", "entry_date": "2025-12-11", "source_folder": "20251211_mini-v1.17.2_gpt-5.2-2025-12-11", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 5.2", "model_org": "OpenAI", "models": [ "gpt-5.2-2025-12-11" ], "model_release_date": "2025-12-11", "open_source_system": true, "open_source_model": false, "resolved_percent": 69, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 134.8337487, "mean_cost_per_instance_usd": 0.2696674974, "mean_api_calls_per_instance": 16.27, "agent_version": "1.17.2", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Nemotron-CORTEXA", "entry_date": "2025-05-16", "source_folder": "20250516_cortexa_o3", "agent": "Nemotron-CORTEXA", "agent_org": "NVIDIA", "model": "Multiple", "model_org": "OpenAI", "models": [ "NV-EmbedCode", "o3", "o3-mini", "o1", "gpt-4.1", "gpt-4o-2024-05-13", "gpt-4-turbo-2024-04-09", "claude-3-7-sonnet-20250219", "deepseek-v3", "Qwen/Qwen2.5-72B-Instruct", "meta-llama/Llama-3.1-405B-Instruct", "meta-llama/Llama-3.3-70B-Instruct" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 68.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://research.nvidia.com/labs/adlr/cortexa/" }, { "entry": "GLM-4.6", "entry_date": "2025-09-30", "source_folder": "20250930_zai_glm4-6", "agent": "Undisclosed", "agent_org": "Z.ai", "model": "GLM-4.6", "model_org": "Z.ai", "models": [ "https://huggingface.co/zai-org/GLM-4.6" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 68.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://chat.z.ai/" }, { "entry": "mini-SWE-agent + Claude 4 Opus (20250514)", "entry_date": "2025-08-02", "source_folder": "20250802_mini-v1.0.0_claude-4-opus-20250514", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Claude 4 Opus", "model_org": "Anthropic", "models": [ "claude-4-opus-20250514" ], "model_release_date": "2025-05-22", "open_source_system": true, "open_source_model": false, "resolved_percent": 67.6, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 565.6352234999999, "mean_cost_per_instance_usd": 1.131270447, "mean_api_calls_per_instance": 31.076, "agent_version": "1.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "SWE-agent + Claude 4 Sonnet", "entry_date": "2025-05-22", "source_folder": "20250522_sweagent_claude-4-sonnet-20250514", "agent": "SWE-agent", "agent_org": "SWE-agent", "model": "Claude 4 Sonnet", "model_org": "Anthropic", "models": [ "claude-4-sonnet-20250514" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 66.6, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/SWE-agent/SWE-agent" }, { "entry": "mini-SWE-agent + Claude 4.5 Haiku (high)", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_claude-4-5-haiku-high", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Claude 4.5 Haiku", "model_org": "Anthropic", "models": [ "claude-haiku-4-5-20251001" ], "model_release_date": "2025-10-15", "open_source_system": true, "open_source_model": false, "resolved_percent": 66.6, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 165.46221805000002, "mean_cost_per_instance_usd": 0.33092443610000005, "mean_api_calls_per_instance": 66.154, "agent_version": "2.0.0", "reasoning_effort": "high", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Aime-coder v1 + Anthopic Claude 3.7 Sonnet", "entry_date": "2025-05-14", "source_folder": "20250514_aime_coder", "agent": "Aime-coder v1", "agent_org": "ByteDance DevInfra", "model": "Claude 3.7 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-7-sonnet-20250219" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 66.4, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/SWE-bench/experiments/blob/main/evaluation/verified/20250514_aime_coder/README.md" }, { "entry": "mini-SWE-agent + GPT 5.1 Codex (medium)", "entry_date": "2025-11-24", "source_folder": "20251124_mini-v1.16.0_gpt-5.1-codex", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 5.1 Codex", "model_org": "OpenAI", "models": [ "gpt-5.1-codex" ], "model_release_date": "2025-11-12", "open_source_system": true, "open_source_model": false, "resolved_percent": 66, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 294.44117500000004, "mean_cost_per_instance_usd": 0.5888823500000001, "mean_api_calls_per_instance": 23.882, "agent_version": "1.16.0", "reasoning_effort": "medium", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + GPT 5.1 (2025-11-13) (medium)", "entry_date": "2025-11-20", "source_folder": "20251120_mini-v1.15.0_gpt-5.1-2025-11-13", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 5.1", "model_org": "OpenAI", "models": [ "gpt-5.1-2025-11-13" ], "model_release_date": "2025-11-12", "open_source_system": true, "open_source_model": false, "resolved_percent": 66, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 153.07773625000002, "mean_cost_per_instance_usd": 0.3061554725, "mean_api_calls_per_instance": 21, "agent_version": "1.15.0", "reasoning_effort": "medium", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "OpenHands", "entry_date": "2025-04-15", "source_folder": "20250415_openhands", "agent": "OpenHands", "agent_org": "OpenHands", "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 65.8, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://all-hands.dev/" }, { "entry": "Augment Agent v0", "entry_date": "2025-03-16", "source_folder": "20250316_augment_agent_v0", "agent": "Augment Agent v0", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 65.4, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.augmentcode.com" }, { "entry": "Amazon Q Developer Agent (v20250405-dev)", "entry_date": "2025-04-05", "source_folder": "20250405_amazon-q-developer-agent-20250405-dev", "agent": "Amazon Q Developer Agent", "agent_org": "AWS", "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 65.4, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://aws.amazon.com/q/developer/" }, { "entry": "OpenHands + Kimi K2", "entry_date": "2025-07-16", "source_folder": "20250716_openhands_kimi_k2", "agent": "OpenHands", "agent_org": "OpenHands", "model": "Kimi K2", "model_org": "Moonshot AI", "models": [ "moonshot/kimi-k2-0711-preview" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 65.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://all-hands.dev/" }, { "entry": "mini-SWE-agent + GPT 5 (2025-08-07) (medium)", "entry_date": "2025-08-07", "source_folder": "20250807_mini-v1.7.0_gpt-5", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 5", "model_org": "OpenAI", "models": [ "gpt-5-2025-08-07" ], "model_release_date": "2025-08-07", "open_source_system": true, "open_source_model": false, "resolved_percent": 65, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 140.19150875, "mean_cost_per_instance_usd": 0.2803830175, "mean_api_calls_per_instance": 13.208, "agent_version": "1.7.0", "reasoning_effort": "medium", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + Claude 4 Sonnet (20250514)", "entry_date": "2025-07-26", "source_folder": "20250726_mini-v1.0.0_claude-sonnet-4-20250514", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Claude 4 Sonnet", "model_org": "Anthropic", "models": [ "claude-4-sonnet-20250514" ], "model_release_date": "2025-05-22", "open_source_system": true, "open_source_model": false, "resolved_percent": 64.93, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 185.7265839, "mean_cost_per_instance_usd": 0.37145316780000004, "mean_api_calls_per_instance": 37.172, "agent_version": "1.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "W&B Programmer O1 crosscheck5", "entry_date": "2025-01-17", "source_folder": "20250117_wandb_programmer_o1_crosscheck5", "agent": "W&B Programmer O1 crosscheck5", "agent_org": null, "model": "o1-preview", "model_org": "OpenAI", "models": [ "o1-preview" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 64.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://wandb.ai" }, { "entry": "PatchPilot-v1.1", "entry_date": "2025-05-03", "source_folder": "20250503_patchpilot-v1.1-o4-mini", "agent": "PatchPilot-v1.1", "agent_org": null, "model": "o4-mini", "model_org": "OpenAI", "models": [ "o4-mini" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 64.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": null }, { "entry": "GLM-4.5", "entry_date": "2025-07-28", "source_folder": "20250728_zai_glm4-5", "agent": "Undisclosed", "agent_org": "Z.ai", "model": "GLM-4.5", "model_org": "Z.ai", "models": [ "https://huggingface.co/zai-org/GLM-4.5" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 64.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://chat.z.ai/" }, { "entry": "AgentScope", "entry_date": "2025-02-06", "source_folder": "20250206_agentscope", "agent": "AgentScope", "agent_org": "Alibaba", "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 63.4, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/modelscope/agentscope" }, { "entry": "mini-SWE-agent + Kimi K2 Thinking", "entry_date": "2025-12-10", "source_folder": "20251210_mini-v1.17.2_kimi-k2-thinking", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Kimi K2 Thinking", "model_org": "Moonshot AI", "models": [ "Kimi-K2-Thinking" ], "model_release_date": "2025-11-06", "open_source_system": true, "open_source_model": true, "resolved_percent": 63.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 219.15361769999998, "mean_cost_per_instance_usd": 0.4383072354, "mean_api_calls_per_instance": 46.82, "agent_version": "1.17.2", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Tools + Claude 3.7 Sonnet (2025-02-24)", "entry_date": "2025-02-24", "source_folder": "20250224_tools_claude-3-7-sonnet", "agent": "Tools", "agent_org": "Anthropic", "model": "Claude 3.7 Sonnet", "model_org": "Anthropic", "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 63.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.anthropic.com/" }, { "entry": "Blackbox AI Agent", "entry_date": "2025-01-10", "source_folder": "20250110_blackboxai_agent_v1.1", "agent": "Blackbox AI Agent", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 62.8, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.blackbox.ai/" }, { "entry": "EPAM AI/Run Developer Agent v20250219 + Anthopic Claude 3.5 Sonnet", "entry_date": "2025-02-28", "source_folder": "20250228_epam-ai-run-claude-3-5-sonnet", "agent": "EPAM AI/Run Developer Agent", "agent_org": null, "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 62.8, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.epam.com/services/artificial-intelligence" }, { "entry": "SWE-agent + Claude 3.7 Sonnet w/ Review Heavy", "entry_date": "2025-02-25", "source_folder": "20250225_sweagent_claude-3-7-sonnet", "agent": "SWE-agent", "agent_org": "SWE-agent", "model": "Claude 3.7 Sonnet w/ Review Heavy", "model_org": "Anthropic", "models": [ "claude-3-7-sonnet-20250219" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 62.4, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/SWE-agent/SWE-agent" }, { "entry": "CodeStory Midwit Agent + swe-search", "entry_date": "2024-12-21", "source_folder": "20241221_codestory_midwit_claude-3-5-sonnet_swe-search", "agent": "CodeStory Midwit Agent", "agent_org": null, "model": "swe-search", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 62.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://aide.dev/" }, { "entry": "mini-SWE-agent + MiniMax M2", "entry_date": "2025-11-24", "source_folder": "20251124_mini-v1.17.0_minimax-m2", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "MiniMax M2", "model_org": "Minimax", "models": [ "minimax-m2" ], "model_release_date": "2025-10-27", "open_source_system": true, "open_source_model": true, "resolved_percent": 61, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 214.128088645, "mean_cost_per_instance_usd": 0.42825617729, "mean_api_calls_per_instance": 74.138, "agent_version": "1.17.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "OpenHands + 4x Scaled (2024-02-03)", "entry_date": "2025-02-03", "source_folder": "20250203_openhands_4x_scaled", "agent": "OpenHands", "agent_org": "OpenHands", "model": "4x Scaled", "model_org": null, "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 60.8, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://all-hands.dev/" }, { "entry": "EntroPO + R2E + Qwen3-Coder-30B-A3B-Instruct", "entry_date": "2025-09-01", "source_folder": "20250901_entroPO_R2E_QwenCoder30BA3B_tts", "agent": "EntroPO + R2E", "agent_org": "42-b3yond-6ug", "model": "Qwen3-Coder-30B-A3B-Instruct", "model_org": "Qwen", "models": [ "Qwen3-Coder-30B-A3B-Instruct" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 60.4, "attempts": "2+", "verified": false, "verified_note": "false (See README.md for info on how to get your results verified)", "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/sherdencooper/R2E-Gym" }, { "entry": "Learn-by-interact", "entry_date": "2025-01-10", "source_folder": "20250110_learn_by_interact_claude3.5", "agent": "Learn-by-interact", "agent_org": "Google", "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 60.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://arxiv.org/pdf/2501.10893" }, { "entry": "mini-SWE-agent + DeepSeek V3.2 Reasoner", "entry_date": "2025-12-01", "source_folder": "20251201_mini-v1.17.1_deepseek-v3.2-reasoner", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "DeepSeek V3.2 Reasoner", "model_org": "deepseek", "models": [ "deepseek-v3.2-reasoner" ], "model_release_date": "2025-12-01", "open_source_system": true, "open_source_model": true, "resolved_percent": 60, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 14.0381514, "mean_cost_per_instance_usd": 0.0280763028, "mean_api_calls_per_instance": 46.404, "agent_version": "1.17.1", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + GPT 5 mini (2025-08-07) (medium)", "entry_date": "2025-08-07", "source_folder": "20250807_mini-v1.7.0_gpt-5-mini", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 5 mini", "model_org": "OpenAI", "models": [ "gpt-5-mini-2025-08-07" ], "model_release_date": "2025-08-07", "open_source_system": true, "open_source_model": false, "resolved_percent": 59.8, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 17.73853365, "mean_cost_per_instance_usd": 0.035477067300000005, "mean_api_calls_per_instance": 14.466, "agent_version": "1.7.0", "reasoning_effort": "medium", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "DeepSWE-Preview + TTS(Bo16)", "entry_date": "2025-06-29", "source_folder": "20250629_deepswerl_r2eagent_tts", "agent": "DeepSWE-Preview", "agent_org": "Agentica", "model": "TTS(Bo16)", "model_org": "OpenAI", "models": [ "https://huggingface.co/agentica-org/DeepSWE-Preview" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 58.8, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "agentica-project.com/" }, { "entry": "mini-SWE-agent + o3 (2025-04-16)", "entry_date": "2025-07-26", "source_folder": "20250726_mini-v1.0.0_o3-2025-04-16", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "o3", "model_org": "OpenAI", "models": [ "o3-20250416" ], "model_release_date": "2025-04-16", "open_source_system": true, "open_source_model": false, "resolved_percent": 58.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 166.826374, "mean_cost_per_instance_usd": 0.333652748, "mean_api_calls_per_instance": 24.698, "agent_version": "1.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Nemotron-CORTEXA", "entry_date": "2025-04-10", "source_folder": "20250410_cortexa", "agent": "Nemotron-CORTEXA", "agent_org": "NVIDIA", "model": "Multiple", "model_org": "Anthropic", "models": [ "NV-EmbedCode", "claude-3-5-sonnet-20241022", "deepseek-v3", "o3-mini", "gpt-4o-2024-05-13", "gpt-4-turbo-2024-04-09", "Qwen/Qwen2.5-72B-Instruct", "meta-llama/Llama-3.1-405B-Instruct", "meta-llama/Llama-3.3-70B-Instruct" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 58.2, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://research.nvidia.com/labs/adlr/cortexa/" }, { "entry": "devlo", "entry_date": "2024-12-13", "source_folder": "20241213_devlo", "agent": "devlo", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 58.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://devlo.ai/?utm_source=swe-bench" }, { "entry": "Emergent E1 (v2024-12-23)", "entry_date": "2024-12-23", "source_folder": "20241223_emergent", "agent": "Emergent E1", "agent_org": null, "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022", "gpt-4o-2024-08-06" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 57.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://emergent.sh" }, { "entry": "Gru(2024-12-08)", "entry_date": "2024-12-08", "source_folder": "20241208_gru", "agent": "Gru", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 57, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://gru.ai" }, { "entry": "Artemis Agent v2 (2025-09-24)", "entry_date": "2025-09-24", "source_folder": "20250924_artemis_agent_v2", "agent": "Artemis Agent v2", "agent_org": "Turintech", "model": "Claude Sonnet 4", "model_org": "Anthropic", "models": [ "claude-sonnet-4-20250514" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 57, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.turintech.ai/" }, { "entry": "SWE-Rizzo", "entry_date": "2025-04-05", "source_folder": "20250405_swe-rizzo_claude37", "agent": "SWE-Rizzo", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 56.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://gen42.ai" }, { "entry": "mini-SWE-agent + Devstral Small (2512)", "entry_date": "2025-12-09", "source_folder": "20251209_mini-v1.17.2_devstral-small-2512", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Devstral Small (2512)", "model_org": "mistral", "models": [ "devstral-small-2512" ], "model_release_date": "2025-12-09", "open_source_system": true, "open_source_model": true, "resolved_percent": 56.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": 86.866, "agent_version": "1.17.2", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + GPT 5 mini", "entry_date": "2026-02-17", "source_folder": "20260217_mini-v2.0.0_gpt-5-mini", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 5 mini", "model_org": "OpenAI", "models": [ "gpt-5-mini-2025-08-07" ], "model_release_date": "2025-08-07", "open_source_system": true, "open_source_model": false, "resolved_percent": 56.2, "attempts": "1", "verified": null, "verified_note": null, "total_cost_usd": 23.600609549999998, "mean_cost_per_instance_usd": 0.0472012191, "mean_api_calls_per_instance": 20.342, "agent_version": "2.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "EPAM AI/Run Developer Agent v20241212 + Anthopic Claude 3.5 Sonnet", "entry_date": "2024-12-12", "source_folder": "20241212_epam-ai-run-claude-3-5-sonnet", "agent": "EPAM AI/Run Developer Agent", "agent_org": null, "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 55.4, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.epam.com/services/artificial-intelligence" }, { "entry": "mini-SWE-agent + Qwen3-Coder 480B/A35B Instruct", "entry_date": "2025-08-02", "source_folder": "20250802_mini-v1.0.0_qwen3-coder-480b-a35b-instruct", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Qwen3-Coder 480B/A35B Instruct", "model_org": "Qwen", "models": [ "Qwen3-Coder-480B-A35B-Instruct" ], "model_release_date": "2025-07-22", "open_source_system": true, "open_source_model": true, "resolved_percent": 55.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 123.963554, "mean_cost_per_instance_usd": 0.247927108, "mean_api_calls_per_instance": 64.792, "agent_version": "1.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + GLM 4.6 (T=1)", "entry_date": "2025-12-01", "source_folder": "20251201_mini-v1.17.1_glm-4.6", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GLM 4.6", "model_org": "Z.ai", "models": [ "glm-4.6" ], "model_release_date": "2025-09-30", "open_source_system": true, "open_source_model": true, "resolved_percent": 55.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 48.3050376, "mean_cost_per_instance_usd": 0.09661007519999999, "mean_api_calls_per_instance": 49.348, "agent_version": "1.17.1", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Amazon Q Developer Agent (v20241202-dev)", "entry_date": "2024-12-02", "source_folder": "20241202_amazon-q-developer-agent-20241202-dev", "agent": "Amazon Q Developer Agent", "agent_org": "AWS", "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 55, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://aws.amazon.com/q/developer/" }, { "entry": "devlo", "entry_date": "2024-11-08", "source_folder": "20241108_devlo", "agent": "devlo", "agent_org": null, "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 54.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://devlo.ai/?utm_source=swe-bench" }, { "entry": "mini-SWE-agent + GLM 4.5 (2025-08-22)", "entry_date": "2025-08-22", "source_folder": "20250822_mini-v1.9.1_glm-4.5", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GLM 4.5", "model_org": "Z.ai", "models": [ "GLM-4.5" ], "model_release_date": "2025-07-28", "open_source_system": true, "open_source_model": true, "resolved_percent": 54.2, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 148.5951744, "mean_cost_per_instance_usd": 0.2971903488, "mean_api_calls_per_instance": 40.224, "agent_version": "1.9.1", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + Devstral (2512)", "entry_date": "2025-12-09", "source_folder": "20251209_mini-v1.17.2_devstral-2512", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Devstral (2512)", "model_org": "mistral", "models": [ "devstral-2512" ], "model_release_date": "2025-12-09", "open_source_system": true, "open_source_model": true, "resolved_percent": 53.8, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": 75.086, "agent_version": "1.17.2", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "FrogBoss-32B-2510", "entry_date": "2025-11-10", "source_folder": "20251110_frogboss-32b", "agent": "FrogBoss-32B-2510", "agent_org": "Microsoft", "model": "Frogboss 32B 2510", "model_org": null, "models": [ "FrogBoss-32B-2510" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 53.6, "attempts": "1", "verified": false, "verified_note": "false (See README.md for info on how to get your results verified)", "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://microsoft.github.io/debug-gym/blog/2025/10/bug-pilot/" }, { "entry": "mini-SWE-agent + Gemini 2.5 Pro (2025-05-06)", "entry_date": "2025-07-26", "source_folder": "20250726_mini-v1.0.0_gemini-2.5-pro", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Gemini 2.5 Pro", "model_org": "Google DeepMind", "models": [ "gemini-2.5-pro" ], "model_release_date": "2025-05-06", "open_source_system": true, "open_source_model": false, "resolved_percent": 53.6, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 144.18604781250002, "mean_cost_per_instance_usd": 0.28837209562500005, "mean_api_calls_per_instance": 20.478, "agent_version": "1.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "CodeSweep - SWE-agent - Kimi K2 Instruct", "entry_date": "2025-08-04", "source_folder": "20250804_codesweep_sweagent_kimi_k2_instruct", "agent": "CodeSweep - SWE-agent", "agent_org": "CodeSweep Inc.", "model": "Kimi K2 Instruct", "model_org": "Moonshot AI", "models": [ "kimi-k2-instruct" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 53.4, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://codesweep.ai/" }, { "entry": "Bracket.sh", "entry_date": "2025-01-20", "source_folder": "20250120_Bracket", "agent": "Bracket.sh", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 53.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://bracket.sh/" }, { "entry": "OpenHands + CodeAct v2.1 (claude-3-5-sonnet-20241022)", "entry_date": "2024-10-29", "source_folder": "20241029_OpenHands-CodeAct-2.1-sonnet-20241022", "agent": "OpenHands", "agent_org": "OpenHands", "model": "CodeAct v2.1 (claude-3-5-sonnet-20241022)", "model_org": "Anthropic", "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 53, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://docs.all-hands.dev/" }, { "entry": "mini-SWE-agent + Claude 3.7 Sonnet (20250219)", "entry_date": "2025-07-20", "source_folder": "20250720_mini-v0.0.0-claude-3-7-sonnet-20250219", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Claude 3.7 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-7-sonnet-20250219" ], "model_release_date": "2025-02-24", "open_source_system": true, "open_source_model": false, "resolved_percent": 52.8, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 177.3077241, "mean_cost_per_instance_usd": 0.3546154482, "mean_api_calls_per_instance": 34.7, "agent_version": "0.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "EntroPO + R2E + Qwen3-Coder-30B-A3B-Instruct", "entry_date": "2025-09-01", "source_folder": "20250901_entroPO_R2E_QwenCoder30BA3B", "agent": "EntroPO + R2E", "agent_org": "42-b3yond-6ug", "model": "Qwen3-Coder-30B-A3B-Instruct", "model_org": "Qwen", "models": [ "Qwen3-Coder-30B-A3B-Instruct" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 52.2, "attempts": "1", "verified": false, "verified_note": "false (See README.md for info on how to get your results verified)", "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/sherdencooper/R2E-Gym" }, { "entry": "Google Jules + Gemini 2.0 Flash (v20241212-experimental)", "entry_date": "2024-12-12", "source_folder": "20241212_google_jules_gemini_2.0_flash_experimental", "agent": "Google Jules", "agent_org": "Google", "model": "Gemini 2.0 Flash (v20241212-experimental)", "model_org": "Google DeepMind", "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 52.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://developers.googleblog.com/en/the-next-chapter-of-the-gemini-era-for-developers/" }, { "entry": "Engine Labs (2024-11-25)", "entry_date": "2024-11-25", "source_folder": "20241125_enginelabs", "agent": "Engine Labs", "agent_org": "Engine Labs", "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 51.8, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.enginelabs.ai/" }, { "entry": "AutoCodeRover-v2.1 (Claude-3.5-Sonnet-20241022)", "entry_date": "2025-01-22", "source_folder": "20250122_autocoderover-v2.1-claude-3-5-sonnet-20241022", "agent": "AutoCodeRover-v2.1", "agent_org": null, "model": "Claude 3.5-Sonnet-20241022", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 51.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.autocoderover.net/" }, { "entry": "OpenHands + Qwen3-Coder-30B-A3B-Instruct", "entry_date": "2025-08-05", "source_folder": "20250805_openhands-Qwen3-Coder-30B-A3B-Instruct", "agent": "OpenHands", "agent_org": "Qwen", "model": "Qwen3-Coder-30B-A3B-Instruct", "model_org": "Qwen", "models": [ "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 51.6, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://qwenlm.github.io/blog/qwen3-coder/" }, { "entry": "Agentless-1.5 + Claude-3.5 Sonnet (20241022)", "entry_date": "2024-12-02", "source_folder": "20241202_agentless-1.5_claude-3.5-sonnet-20241022", "agent": "Agentless-1.5", "agent_org": "Agentless", "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 50.8, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/OpenAutoCoder/Agentless" }, { "entry": "Solver (2024-10-28)", "entry_date": "2024-10-28", "source_folder": "20241028_solver", "agent": "Solver", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 50, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://solverai.com/" }, { "entry": "Bytedance MarsCode Agent", "entry_date": "2024-11-25", "source_folder": "20241125_marscode-agent-dev", "agent": "Bytedance MarsCode Agent", "agent_org": "Bytedance", "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 50, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.marscode.com/" }, { "entry": "nFactorial (2024-11-05)", "entry_date": "2024-11-05", "source_folder": "20241105_nfactorial", "agent": "nFactorial", "agent_org": null, "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022", "gpt-4o-2024-08-06" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 49.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://nfactorial.dev/" }, { "entry": "Tools + Claude 3.5 Sonnet (2024-10-22)", "entry_date": "2024-10-22", "source_folder": "20241022_tools_claude-3-5-sonnet-updated", "agent": "Tools", "agent_org": "Anthropic", "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022", "claude-3.5-sonnet-20241022" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 49, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.anthropic.com/" }, { "entry": "Composio SWE-Kit (2024-10-25)", "entry_date": "2024-10-25", "source_folder": "20241025_composio_swekit", "agent": "Composio SWE-Kit", "agent_org": null, "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022", "o1-mini" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 48.6, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/ComposioHQ/composio/tree/master/python/swe/agent" }, { "entry": "AppMap Navie v2", "entry_date": "2024-11-06", "source_folder": "20241106_navie-2-gpt4o-sonnet", "agent": "AppMap Navie v2", "agent_org": null, "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022", "gpt-4o-2024-08-06" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 47.2, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://appmap.io/navie" }, { "entry": "Skywork-SWE-32B + TTS(Bo8)", "entry_date": "2025-06-16", "source_folder": "20250616_Skywork-SWE-32B+TTS_Bo8", "agent": "Skywork-SWE-32B", "agent_org": "Skywork AI", "model": "TTS(Bo8)", "model_org": "Qwen", "models": [ "Qwen2.5-Coder-32B-Instruct" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 47, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://quixotic-sting-239.notion.site/eb17f379610040ceb54da5d5d24065bd" }, { "entry": "OpenHands + DevStral Small 2505", "entry_date": "2025-05-20", "source_folder": "20250520_openhands_devstral_small", "agent": "OpenHands", "agent_org": "OpenHands", "model": "DevStral Small 2505", "model_org": "Mistral", "models": [ "mistralai/Devstral-Small-2505" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 46.8, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://all-hands.dev/" }, { "entry": "Emergent E1 (v2024-10-12)", "entry_date": "2024-10-23", "source_folder": "20241023_emergent", "agent": "Emergent E1", "agent_org": null, "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022", "gpt-4o-2024-08-06" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 46.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://emergent.sh" }, { "entry": "AutoCodeRover-v2.0 (Claude-3.5-Sonnet-20241022)", "entry_date": "2024-11-08", "source_folder": "20241108_autocoderover-v2.0-claude-3-5-sonnet-20241022", "agent": "AutoCodeRover-v2.0", "agent_org": null, "model": "Claude 3.5-Sonnet-20241022", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 46.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.autocoderover.net/" }, { "entry": "PatchPilot + Co-PatcheR", "entry_date": "2025-05-28", "source_folder": "20250528_patchpilot_Co-PatcheR", "agent": "PatchPilot", "agent_org": null, "model": "Co-PatcheR", "model_org": null, "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 46, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/ucsb-mlsec/Co-PatcheR" }, { "entry": "Solver (2024-09-12)", "entry_date": "2024-09-24", "source_folder": "20240924_solver", "agent": "Solver", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 45.4, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://laredolabs.com/" }, { "entry": "Gru(2024-08-24)", "entry_date": "2024-08-24", "source_folder": "20240824_gru", "agent": "Gru", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 45.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://gru.ai" }, { "entry": "FrogMini-14B-2510", "entry_date": "2025-11-10", "source_folder": "20251110_frogmini-14b", "agent": "FrogMini-14B-2510", "agent_org": "Microsoft", "model": "Frogmini 14B 2510", "model_org": null, "models": [ "FrogMini-14B-2510" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 45, "attempts": "1", "verified": false, "verified_note": "false (See README.md for info on how to get your results verified)", "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://microsoft.github.io/debug-gym/blog/2025/10/bug-pilot/" }, { "entry": "mini-SWE-agent + o4-mini (2025-04-16)", "entry_date": "2025-07-26", "source_folder": "20250726_mini-v1.0.0_o4-mini-2025-04-16", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "o4-mini", "model_org": "OpenAI", "models": [ "o4-mini-20250416" ], "model_release_date": "2025-04-16", "open_source_system": true, "open_source_model": false, "resolved_percent": 45, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 104.9866664, "mean_cost_per_instance_usd": 0.2099733328, "mean_api_calls_per_instance": 23.278, "agent_version": "1.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "CodeShellAgent + Gemini 2.0 Flash (Experimental)", "entry_date": "2025-01-18", "source_folder": "20250118_codeshellagent_gemini_2.0_flash_experimental", "agent": "CodeShellAgent", "agent_org": null, "model": "Gemini 2.0 Flash (Experimental)", "model_org": "Google DeepMind", "models": [ "gemini-2.0-flash-exp" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 44.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/WisdomShell/codeshell" }, { "entry": "mini-SWE-agent + Kimi K2 Instruct", "entry_date": "2025-08-07", "source_folder": "20250807_mini-v1.7.0_kimi-k2-instruct", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Kimi K2 Instruct", "model_org": "Moonshot AI", "models": [ "Kimi-K2-Instruct" ], "model_release_date": "2025-07-11", "open_source_system": true, "open_source_model": true, "resolved_percent": 43.8, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 265.796683, "mean_cost_per_instance_usd": 0.531593366, "mean_api_calls_per_instance": 37.52, "agent_version": "1.7.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Solver (2024-09-12)", "entry_date": "2024-09-20", "source_folder": "20240920_solver", "agent": "Solver", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 43.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://laredolabs.com/" }, { "entry": "Amazon Nova Premier 1.0 (2025-04-30)", "entry_date": "2025-05-27", "source_folder": "20250527_amazon.nova-premier-v1.0", "agent": "Amazon Nova Premier 1.0", "agent_org": "Amazon Nova", "model": "Amazon.nova Premier v1:0", "model_org": null, "models": [ "amazon.nova-premier-v1:0" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 42.4, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "aws.amazon.com/nova" }, { "entry": "Agentless Lite + O3 Mini (20250214)", "entry_date": "2025-02-14", "source_folder": "20250214_agentless_lite_o3_mini", "agent": "Agentless Lite", "agent_org": "Agentless", "model": "O3 Mini", "model_org": "OpenAI", "models": [ "o3-mini" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 42.4, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/sorendunn/Agentless-Lite" }, { "entry": "DeepSWE-Preview", "entry_date": "2025-06-29", "source_folder": "20250629_deepswerl_r2eagent", "agent": "R2E-Gym", "agent_org": "Agentica", "model": "DeepSWE-Preview", "model_org": null, "models": [ "https://huggingface.co/agentica-org/DeepSWE-Preview" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 42.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "agentica-project.com/" }, { "entry": "SWE-Exp", "entry_date": "2025-08-06", "source_folder": "20250806_SWE-Exp_DeepSeek-V3", "agent": "SWE-Exp", "agent_org": "SWE-Exp", "model": "DeepSeek V3 0324", "model_org": "DeepSeek", "models": [ "DeepSeek-V3-0324" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 42, "attempts": "2", "verified": false, "verified_note": "false (See README.md for info on how to get your results verified)", "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/YerbaPage/SWE-Exp" }, { "entry": "ugaiforge", "entry_date": "2025-01-12", "source_folder": "20250112_ugaiforge", "agent": "ugaiforge", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 41.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": null }, { "entry": "nFactorial (2024-10-30)", "entry_date": "2024-10-30", "source_folder": "20241030_nfactorial", "agent": "nFactorial", "agent_org": null, "model": "Multiple", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022", "gpt-4o-2024-08-06", "o1-preview" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 41.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://nfactorial.dev/" }, { "entry": "SWE-RL (Llama3-SWE-RL-70B + Agentless Mini) (20250226)", "entry_date": "2025-02-26", "source_folder": "20250226_swerl_llama3_70b", "agent": "Agentless Mini", "agent_org": "Agentless", "model": "Llama3-SWE-RL-70B", "model_org": "Meta", "models": [ "Llama3-SWE-RL-70B" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 41.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/facebookresearch/swe-rl" }, { "entry": "Nebius AI Qwen 2.5 72B Generator + LLama 3.1 70B Critic", "entry_date": "2024-11-13", "source_folder": "20241113_nebius-search-open-weight-models-11-24", "agent": "Nebius AI", "agent_org": null, "model": "Multiple", "model_org": "Meta", "models": [ "Llama 3.1", "Qwen 2.5" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 40.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://nebius.com/blog/posts/training-and-search-for-software-engineering-agents" }, { "entry": "Tools + Claude 3.5 Haiku", "entry_date": "2024-10-22", "source_folder": "20241022_tools_claude-3-5-haiku", "agent": "Tools", "agent_org": "Anthropic", "model": "Claude 3.5 Haiku", "model_org": "Anthropic", "models": [ "claude-3-haiku-20240307" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 40.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.anthropic.com/" }, { "entry": "Honeycomb", "entry_date": "2024-08-20", "source_folder": "20240820_honeycomb", "agent": "Honeycomb", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 40.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://honeycomb.sh/" }, { "entry": "Composio SWEkit + Claude 3.5 Sonnet (2024-10-16)", "entry_date": "2024-10-16", "source_folder": "20241016_composio_swekit", "agent": "Composio SWEkit", "agent_org": null, "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022", "claude-3.5-sonnet-20241022" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 40.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/ComposioHQ/composio/tree/master/python/swe/agent" }, { "entry": "SWE-agent + SWE-agent-LM-32B", "entry_date": "2025-05-11", "source_folder": "20250511_sweagent_lm_32b", "agent": "SWE-agent", "agent_org": "SWE-agent", "model": "SWE-agent-LM-32B", "model_org": "Qwen", "models": [ "Qwen 2.5" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 40.2, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://swesmith.com/" }, { "entry": "EPAM AI/Run Developer Agent v20241029 + Anthopic Claude 3.5 Sonnet", "entry_date": "2024-10-29", "source_folder": "20241029_epam-ai-run-claude-3-5-sonnet", "agent": "EPAM AI/Run Developer Agent", "agent_org": null, "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 39.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.epam.com/services/artificial-intelligence" }, { "entry": "mini-SWE-agent + GPT 4.1 (2025-04-14)", "entry_date": "2025-07-26", "source_folder": "20250726_mini-v1.0.0_gpt-4.1-2025-04-14", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 4.1", "model_org": "OpenAI", "models": [ "gpt-4.1-20250414" ], "model_release_date": "2025-04-14", "open_source_system": true, "open_source_model": false, "resolved_percent": 39.58, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 73.156562, "mean_cost_per_instance_usd": 0.146313124, "mean_api_calls_per_instance": 19.544, "agent_version": "1.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Amazon Q Developer Agent (v20240719-dev)", "entry_date": "2024-07-21", "source_folder": "20240721_amazon-q-developer-agent-20240719-dev", "agent": "Amazon Q Developer Agent", "agent_org": "AWS", "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 38.8, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://aws.amazon.com/q/developer/" }, { "entry": "Agentless-1.5 + GPT 4o (2024-05-13)", "entry_date": "2024-10-28", "source_folder": "20241028_agentless-1.5_gpt4o", "agent": "Agentless-1.5", "agent_org": "Agentless", "model": "GPT-4o", "model_org": "OpenAI", "models": [ "gpt-4o-2024-05-13" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 38.8, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/OpenAutoCoder/Agentless" }, { "entry": "AutoCodeRover (v20240620) + GPT 4o (2024-05-13)", "entry_date": "2024-06-28", "source_folder": "20240628_autocoderover-v20240620", "agent": "AutoCodeRover", "agent_org": null, "model": "GPT-4o", "model_org": "OpenAI", "models": [ "gpt-4o-2024-05-13" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 38.4, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://autocoderover.dev/" }, { "entry": "Skywork-SWE-32B", "entry_date": "2025-06-16", "source_folder": "20250616_Skywork-SWE-32B", "agent": "Skywork-SWE-32B", "agent_org": "Skywork AI", "model": "Qwen2.5 Coder 32B Instruct", "model_org": "Qwen", "models": [ "Qwen2.5-Coder-32B-Instruct" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 38, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://quixotic-sting-239.notion.site/eb17f379610040ceb54da5d5d24065bd" }, { "entry": "SWE-agent + DevStral Small 2507", "entry_date": "2025-07-25", "source_folder": "20250725_sweagent_devstral_small_2507", "agent": "SWE-agent", "agent_org": "SWE-agent", "model": "DevStral Small 2507", "model_org": "Mistral", "models": [ "devstral-small-2507" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 38, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://swe-agent.com/" }, { "entry": "Factory Code Droid", "entry_date": "2024-06-17", "source_folder": "20240617_factory_code_droid", "agent": "Factory Code Droid", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 37, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.factory.ai/" }, { "entry": "mini-SWE-agent + GPT 5 nano (2025-08-07) (medium)", "entry_date": "2025-08-07", "source_folder": "20250807_mini-v1.7.0_gpt-5-nano", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 5 nano", "model_org": "OpenAI", "models": [ "gpt-5-nano-2025-08-07" ], "model_release_date": "2025-08-07", "open_source_system": true, "open_source_model": false, "resolved_percent": 34.8, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 19.037590390000002, "mean_cost_per_instance_usd": 0.03807518078, "mean_api_calls_per_instance": 39.76, "agent_version": "1.7.0", "reasoning_effort": "medium", "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "SWE-agent + Claude 3.5 Sonnet", "entry_date": "2024-06-20", "source_folder": "20240620_sweagent_claude3.5sonnet", "agent": "SWE-agent", "agent_org": "SWE-agent", "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "models": [ "claude-3-5-sonnet-20241022" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 33.6, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/SWE-agent/SWE-agent" }, { "entry": "SWE-Fixer (Qwen2.5-7b retriever + Qwen2.5-72b editor)", "entry_date": "2025-03-06", "source_folder": "20250306_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor", "agent": "SWE-Fixer", "agent_org": null, "model": "Qwen2.5 (7B + 72B)", "model_org": "Qwen", "models": [ "Qwen 2.5" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 32.8, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/InternLM/SWE-Fixer" }, { "entry": "MASAI + GPT 4o (2024-06-12)", "entry_date": "2024-06-12", "source_folder": "20240612_MASAI_gpt4o", "agent": "MASAI", "agent_org": null, "model": "GPT-4o", "model_org": "OpenAI", "models": [ "gpt-4o-2024-08-06" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 32.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://masai-dev-agent.github.io/" }, { "entry": "Artemis Agent v1 (2024-11-20)", "entry_date": "2024-11-20", "source_folder": "20241120_artemis_agent", "agent": "Artemis Agent v1", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 32, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.turintech.ai/" }, { "entry": "nFactorial (2024-10-07)", "entry_date": "2024-10-07", "source_folder": "20241007_nfactorial", "agent": "nFactorial", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 31.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://nfactorial.dev/" }, { "entry": "SWE-Fixer (Qwen2.5-7b retriever + Qwen2.5-72b editor) 20241128", "entry_date": "2024-11-28", "source_folder": "20241128_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor_20241128", "agent": "SWE-Fixer", "agent_org": null, "model": "Qwen2.5 (7B + 72B)", "model_org": "Qwen", "models": [ "Qwen 2.5" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 30.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/InternLM/SWE-Fixer" }, { "entry": "Lingma Agent + Lingma SWE-GPT 72b (v0925)", "entry_date": "2024-10-02", "source_folder": "20241002_lingma-agent_lingma-swe-gpt-72b", "agent": "Lingma Agent", "agent_org": "Alibaba", "model": "Lingma SWE-GPT 72b (v0925)", "model_org": "OpenAI", "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 28.8, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT (https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT-v20240925)" }, { "entry": "mini-SWE-agent + Gemini 2.5 Flash (2025-04-17)", "entry_date": "2025-07-26", "source_folder": "20250726_mini-v1.0.0_gemini-2.5-flash", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Gemini 2.5 Flash", "model_org": "Google DeepMind", "models": [ "gemini-2.5-flash" ], "model_release_date": "2025-04-17", "open_source_system": true, "open_source_model": false, "resolved_percent": 28.73, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 66.942859175, "mean_cost_per_instance_usd": 0.13388571835, "mean_api_calls_per_instance": 26.646, "agent_version": "1.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "EPAM AI/Run Developer Agent + GPT4o", "entry_date": "2024-10-16", "source_folder": "20241016_epam-ai-run-gpt-4o", "agent": "EPAM AI/Run Developer Agent", "agent_org": null, "model": "GPT-4o", "model_org": "OpenAI", "models": [ "gpt-4o-2024-08-06" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 27, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.epam.com/services/artificial-intelligence" }, { "entry": "AppMap Navie + GPT 4o (2024-05-13)", "entry_date": "2024-06-15", "source_folder": "20240615_appmap-navie_gpt4o", "agent": "AppMap Navie", "agent_org": null, "model": "GPT-4o", "model_org": "OpenAI", "models": [ "gpt-4o-2024-05-13" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 26.2, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://appmap.io/navie" }, { "entry": "mini-SWE-agent + gpt-oss-120b", "entry_date": "2025-08-07", "source_folder": "20250807_mini-v1.7.0_gpt-oss-120b", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "gpt-oss-120b", "model_org": "OpenAI", "models": [ "gpt-oss-120b" ], "model_release_date": "2025-08-05", "open_source_system": true, "open_source_model": true, "resolved_percent": 26, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 28.559527899999996, "mean_cost_per_instance_usd": 0.05711905579999999, "mean_api_calls_per_instance": 27.598, "agent_version": "1.7.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "nFactorial (2024-10-01)", "entry_date": "2024-10-01", "source_folder": "20241001_nfactorial", "agent": "nFactorial", "agent_org": null, "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 25.8, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://nfactorial.dev/" }, { "entry": "Amazon Q Developer Agent (v20240430-dev)", "entry_date": "2024-05-09", "source_folder": "20240509_amazon-q-developer-agent-20240430-dev", "agent": "Amazon Q Developer Agent", "agent_org": "AWS", "model": "Undisclosed", "model_org": null, "models": [], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 25.6, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://aws.amazon.com/q/developer/" }, { "entry": "Lingma Agent + Lingma SWE-GPT 72b (v0918)", "entry_date": "2024-09-18", "source_folder": "20240918_lingma-agent_lingma-swe-gpt-72b", "agent": "Lingma Agent", "agent_org": "Alibaba", "model": "Lingma SWE-GPT 72b (v0918)", "model_org": "OpenAI", "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 25, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT" }, { "entry": "EPAM AI/Run Developer Agent + GPT4o", "entry_date": "2024-08-20", "source_folder": "20240820_epam-ai-run-gpt-4o", "agent": "EPAM AI/Run Developer Agent", "agent_org": null, "model": "GPT-4o", "model_org": "OpenAI", "models": [ "gpt-4o-2024-08-06" ], "model_release_date": null, "open_source_system": false, "open_source_model": false, "resolved_percent": 24, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.epam.com/services/artificial-intelligence" }, { "entry": "mini-SWE-agent + GPT 4.1 mini (2025-04-14)", "entry_date": "2025-07-20", "source_folder": "20250720_mini-v0.0.0_gpt-4.1-mini-2025-04-14", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 4.1 mini", "model_org": "OpenAI", "models": [ "gpt-4.1-mini-20250414" ], "model_release_date": "2025-04-14", "open_source_system": true, "open_source_model": false, "resolved_percent": 23.94, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 217.57707639999998, "mean_cost_per_instance_usd": 0.43515415279999997, "mean_api_calls_per_instance": 89.196, "agent_version": "0.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "MCTS-Refine-7B", "entry_date": "2025-06-27", "source_folder": "20250627_agentless_MCTS-Refine-7B", "agent": "MCTS-Refine-7B", "agent_org": null, "model": "MCTS Refine 7B", "model_org": null, "models": [ "MCTS-Refine-7B" ], "model_release_date": null, "open_source_system": true, "open_source_model": true, "resolved_percent": 23.2, "attempts": "1", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://mcts-refine.github.io/MCTS-Refine" }, { "entry": "SWE-agent + GPT 4o (2024-05-13)", "entry_date": "2024-07-28", "source_folder": "20240728_sweagent_gpt4o", "agent": "SWE-agent", "agent_org": "SWE-agent", "model": "GPT-4o", "model_org": "OpenAI", "models": [ "gpt-4o-2024-05-13" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 23.2, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/SWE-agent/SWE-agent" }, { "entry": "SWE-agent + GPT 4 (1106)", "entry_date": "2024-04-02", "source_folder": "20240402_sweagent_gpt4", "agent": "SWE-agent", "agent_org": "SWE-agent", "model": "GPT-4 (1106)", "model_org": "OpenAI", "models": [ "gpt-4-1106-preview" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 22.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/SWE-agent/SWE-agent" }, { "entry": "mini-SWE-agent + GPT 4o (2024-11-20)", "entry_date": "2025-07-20", "source_folder": "20250720_mini-v0.0.0-gpt-4o-2024-11-20", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "GPT 4o", "model_org": "OpenAI", "models": [ "gpt-4o-20241120" ], "model_release_date": "2024-11-20", "open_source_system": true, "open_source_model": false, "resolved_percent": 21.62, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 765.5740625000001, "mean_cost_per_instance_usd": 1.531148125, "mean_api_calls_per_instance": 63.066, "agent_version": "0.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + Llama 4 Maverick Instruct", "entry_date": "2025-07-20", "source_folder": "20250720_mini-v0.0.0-Llama-4-Maverick-17B-Instruct", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Llama 4 Maverick Instruct", "model_org": "Meta", "models": [ "llama-4-maverick-instruct" ], "model_release_date": "2025-04-05", "open_source_system": true, "open_source_model": false, "resolved_percent": 21.04, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 156.85304879999998, "mean_cost_per_instance_usd": 0.31370609759999996, "mean_api_calls_per_instance": 48.076, "agent_version": "0.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Lingma Agent + Lingma SWE-GPT 7b (v0925)", "entry_date": "2024-10-02", "source_folder": "20241002_lingma-agent_lingma-swe-gpt-7b", "agent": "Lingma Agent", "agent_org": "Alibaba", "model": "Lingma SWE-GPT 7b (v0925)", "model_org": "OpenAI", "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 18.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT (https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT-v20240925)" }, { "entry": "SWE-agent + Claude 3 Opus", "entry_date": "2024-04-02", "source_folder": "20240402_sweagent_claude3opus", "agent": "SWE-agent", "agent_org": "SWE-agent", "model": "Claude 3 Opus", "model_org": "Anthropic", "models": [ "claude-3-opus-20240229" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 15.8, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/SWE-agent/SWE-agent" }, { "entry": "mini-SWE-agent + Gemini 2.0 Flash", "entry_date": "2025-07-26", "source_folder": "20250726_mini-v1.0.0_gemini-2.0-flash", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Gemini 2.0 Flash", "model_org": "Google DeepMind", "models": [ "gemini-2.0-flash" ], "model_release_date": "2025-02-05", "open_source_system": true, "open_source_model": false, "resolved_percent": 13.52, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 0, "mean_cost_per_instance_usd": 0, "mean_api_calls_per_instance": 0, "agent_version": "0.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "Lingma Agent + Lingma SWE-GPT 7b (v0918)", "entry_date": "2024-09-18", "source_folder": "20240918_lingma-agent_lingma-swe-gpt-7b", "agent": "Lingma Agent", "agent_org": "Alibaba", "model": "Lingma SWE-GPT 7b (v0918)", "model_org": "OpenAI", "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 10.2, "attempts": "2+", "verified": false, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT" }, { "entry": "mini-SWE-agent + Llama 4 Scout Instruct", "entry_date": "2025-07-20", "source_folder": "20250720_mini-v0.0.0-Llama-4-Scout-17B-Instruct", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Llama 4 Scout Instruct", "model_org": "Meta", "models": [ "llama-4-scout-instruct" ], "model_release_date": "2025-04-05", "open_source_system": true, "open_source_model": false, "resolved_percent": 9.06, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 57.8157615, "mean_cost_per_instance_usd": 0.115631523, "mean_api_calls_per_instance": 29.216, "agent_version": "0.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "mini-SWE-agent + Qwen2.5-Coder 32B Instruct", "entry_date": "2025-08-03", "source_folder": "20250803_mini-v1.0.0_qwen2-5-coder-32b-instruct", "agent": "mini-SWE-agent", "agent_org": "SWE-agent", "model": "Qwen2.5-Coder 32B Instruct", "model_org": "Qwen", "models": [ "Qwen2.5-Coder-32B-Instruct" ], "model_release_date": "2024-11-12", "open_source_system": true, "open_source_model": true, "resolved_percent": 9, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": 34.05807032999999, "mean_cost_per_instance_usd": 0.06811614065999999, "mean_api_calls_per_instance": 48.206, "agent_version": "1.0.0", "reasoning_effort": null, "system_url": "https://mini-swe-agent.com/latest/" }, { "entry": "RAG + Claude 3 Opus", "entry_date": "2024-04-02", "source_folder": "20240402_rag_claude3opus", "agent": "RAG baseline", "agent_org": null, "model": "Claude 3 Opus", "model_org": "Anthropic", "models": [ "claude-3-opus-20240229" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 7, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": "https://github.com/SWE-bench/SWE-bench/tree/main/swebench/inference" }, { "entry": "RAG + Claude 2", "entry_date": "2023-10-10", "source_folder": "20231010_rag_claude2", "agent": "RAG baseline", "agent_org": null, "model": "Claude 2", "model_org": "Anthropic", "models": [ "claude-2" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 4.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": null }, { "entry": "RAG + GPT 4 (1106)", "entry_date": "2024-04-02", "source_folder": "20240402_rag_gpt4", "agent": "RAG baseline", "agent_org": null, "model": "GPT-4 (1106)", "model_org": "OpenAI", "models": [ "gpt-4-1106-preview" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 2.8, "attempts": "2+", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": null }, { "entry": "RAG + SWE-Llama 7B", "entry_date": "2023-10-10", "source_folder": "20231010_rag_swellama7b", "agent": "RAG baseline", "agent_org": "SWE-agent", "model": "SWE-Llama 7B", "model_org": "Meta", "models": [ "SWE-Llama" ], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 1.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": null }, { "entry": "RAG + SWE-Llama 13B", "entry_date": "2023-10-10", "source_folder": "20231010_rag_swellama13b", "agent": "RAG baseline", "agent_org": "SWE-agent", "model": "SWE-Llama 13B", "model_org": "Meta", "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 1.2, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": null }, { "entry": "RAG + ChatGPT 3.5", "entry_date": "2023-10-10", "source_folder": "20231010_rag_gpt35", "agent": "RAG baseline", "agent_org": "SWE-agent", "model": "GPT-3.5", "model_org": "OpenAI", "models": [], "model_release_date": null, "open_source_system": true, "open_source_model": false, "resolved_percent": 0.4, "attempts": "1", "verified": true, "verified_note": null, "total_cost_usd": null, "mean_cost_per_instance_usd": null, "mean_api_calls_per_instance": null, "agent_version": null, "reasoning_effort": null, "system_url": null } ] }