{
  "dataset": "repeated-attempt-variance",
  "schema_version": "yylo_seo_variance_summary.v1",
  "dataset_version": 1,
  "description": "Derived repeated-attempt variance evidence: every declared system identity submitted two or more times to the public SWE-bench Verified leaderboard, with observed resolved-percentage spreads, per-group identity disclosures, and explicit limitations.",
  "publisher": "yylo.dev",
  "canonical_url": "https://yylo.dev/guides/repeated-attempt-variance-data",
  "download_url": "https://yylo.dev/data/repeated-attempt-variance-v1.json",
  "derived_from": {
    "dataset": "swebench-verified-longitudinal",
    "dataset_version": 1,
    "download_url": "https://yylo.dev/data/swebench-verified-longitudinal-v1.json",
    "canonical_url": "https://yylo.dev/guides/longitudinal-coding-agent-benchmark-dataset",
    "entries_sha256": "sha256:cbd2393a235c0276211fb4d7035c943ef7eb7e4be4902be99a5733f6380d02b6",
    "source_fetched_at": "2026-08-29",
    "source_surface_last_modified": "2026-08-10"
  },
  "method": {
    "identity": "declared system identity: the entry's published agent and model display names, exactly as listed",
    "unit": "one leaderboard entry is one evaluated submission of one declared identity against the fixed 500-instance Verified task set",
    "inclusion": "only declared identities with two or more entries in the parent extract are kept; single-listing identities carry no spread",
    "spread": "the group's highest minus its lowest resolved_percent, computed in integer tenths of a percentage point",
    "median": "the median of an even number of group spreads is the mean of the two middle values",
    "order": "groups ordered by spread_pp descending, then group_id ascending; a group's entries by entry_date ascending, then source_folder ascending",
    "computed_at": "2026-08-29"
  },
  "population": {
    "groups": 18,
    "entries_in_groups": 41,
    "parent_entries": 180,
    "distinct_identities": 157
  },
  "summary": {
    "spreads_pp": {
      "min": 0.8,
      "median": 5.2,
      "max": 39.8
    },
    "observed_percent_range": {
      "min": 24,
      "max": 76.8
    },
    "groups_with_same_date_pairing": 2,
    "groups_with_consistent_attempt_policy": 15,
    "groups_with_consistent_model_identifiers": 12,
    "groups_with_scaffold_version_change": 4
  },
  "field_notes": {
    "group_id": "stable slug derived from the published agent and model display names",
    "min_percent": "the group's lowest published resolved_percent, exactly as republished by the parent extract",
    "max_percent": "the group's highest published resolved_percent, exactly as republished by the parent extract",
    "spread_pp": "observed spread in percentage points: max_percent minus min_percent in integer tenths; an upper bound on run-to-run variance, never an isolated measurement of it",
    "listing_span_days": "days between the group's earliest and latest entry_date; listing dates, not run dates",
    "same_date_pairing": "true when two or more of the group's entries share one listing date",
    "attempt_policies": "distinct published attempt-policy tags inside the group, ascending",
    "attempt_policy_consistent": "true when every entry in the group carries the same attempt-policy tag",
    "model_identifiers": "distinct published model identifier arrays inside the group, sorted by canonical JSON",
    "model_identifiers_consistent": "true when every entry in the group publishes the same model identifier array",
    "scaffold_versions": "distinct non-null published scaffold (mini-SWE-agent) versions inside the group, ascending; empty when the board records none",
    "scaffold_version_changed": "true when the group publishes more than one scaffold version",
    "entries": "the group's submissions in derivation order, republishing only the fields the summary reads",
    "groups_sha256": "SHA-256 over the groups array in canonical form (recursively key-sorted JSON, no whitespace)",
    "verified": "the parent extract's normalized source verification marker; null when the source carries none"
  },
  "limitations": [
    "Declared identity, not a pinned system: submitters may change code, prompts, configuration, or provider behavior between submissions; the board pins the task set, never the system.",
    "Observed spread conflates every such change with sampling noise; it is an upper bound on run-to-run variance, not an isolated measurement of it.",
    "Entry dates are listing dates; a run precedes its listing by an unknown margin, so spans order publication, not measurement.",
    "All entries are third-party self-reported results; nothing in this summary measures any YYLO product or any model the publishers did not run.",
    "Attempt policies, ensemble compositions, scaffold generations, and reasoning-effort tags change inside some groups; the per-group fields record exactly which groups are affected.",
    "Verification is mixed: each entry carries the parent extract's verified marker, and no row is upgraded by appearing here.",
    "The summary is a derived view of dataset version 1 of the parent extract; a parent refresh publishes a new summary version with new hashes and leaves this version's claims intact."
  ],
  "groups_sha256": "sha256:80500ffc69343bbe2dbd67ae7f79ef7f4bec6a154c562aa6f8edbebf516d1306",
  "groups": [
    {
      "group_id": "amazon-q-developer-agent-undisclosed",
      "agent": "Amazon Q Developer Agent",
      "model": "Undisclosed",
      "agent_org": "AWS",
      "n_entries": 4,
      "min_percent": 25.6,
      "max_percent": 65.4,
      "spread_pp": 39.8,
      "listing_span_days": 331,
      "same_date_pairing": false,
      "attempt_policies": [
        "2+"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        []
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2024-05-09",
          "resolved_percent": 25.6,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20240509_amazon-q-developer-agent-20240430-dev"
        },
        {
          "entry_date": "2024-07-21",
          "resolved_percent": 38.8,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20240721_amazon-q-developer-agent-20240719-dev"
        },
        {
          "entry_date": "2024-12-02",
          "resolved_percent": 55,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241202_amazon-q-developer-agent-20241202-dev"
        },
        {
          "entry_date": "2025-04-05",
          "resolved_percent": 65.4,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250405_amazon-q-developer-agent-20250405-dev"
        }
      ]
    },
    {
      "group_id": "epam-ai-run-developer-agent-claude-3-5-sonnet",
      "agent": "EPAM AI/Run Developer Agent",
      "model": "Claude 3.5 Sonnet",
      "agent_org": null,
      "n_entries": 3,
      "min_percent": 39.6,
      "max_percent": 62.8,
      "spread_pp": 23.2,
      "listing_span_days": 122,
      "same_date_pairing": false,
      "attempt_policies": [
        "2+"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "claude-3-5-sonnet-20241022"
        ]
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2024-10-29",
          "resolved_percent": 39.6,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241029_epam-ai-run-claude-3-5-sonnet"
        },
        {
          "entry_date": "2024-12-12",
          "resolved_percent": 55.4,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241212_epam-ai-run-claude-3-5-sonnet"
        },
        {
          "entry_date": "2025-02-28",
          "resolved_percent": 62.8,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250228_epam-ai-run-claude-3-5-sonnet"
        }
      ]
    },
    {
      "group_id": "gru-undisclosed",
      "agent": "Gru",
      "model": "Undisclosed",
      "agent_org": null,
      "n_entries": 2,
      "min_percent": 45.2,
      "max_percent": 57,
      "spread_pp": 11.8,
      "listing_span_days": 106,
      "same_date_pairing": false,
      "attempt_policies": [
        "2+"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        []
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2024-08-24",
          "resolved_percent": 45.2,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20240824_gru"
        },
        {
          "entry_date": "2024-12-08",
          "resolved_percent": 57,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241208_gru"
        }
      ]
    },
    {
      "group_id": "emergent-e1-multiple",
      "agent": "Emergent E1",
      "model": "Multiple",
      "agent_org": null,
      "n_entries": 2,
      "min_percent": 46.6,
      "max_percent": 57.2,
      "spread_pp": 10.6,
      "listing_span_days": 61,
      "same_date_pairing": false,
      "attempt_policies": [
        "2+"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "claude-3-5-sonnet-20241022",
          "gpt-4o-2024-08-06"
        ]
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2024-10-23",
          "resolved_percent": 46.6,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241023_emergent"
        },
        {
          "entry_date": "2024-12-23",
          "resolved_percent": 57.2,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241223_emergent"
        }
      ]
    },
    {
      "group_id": "nemotron-cortexa-multiple",
      "agent": "Nemotron-CORTEXA",
      "model": "Multiple",
      "agent_org": "NVIDIA",
      "n_entries": 2,
      "min_percent": 58.2,
      "max_percent": 68.2,
      "spread_pp": 10,
      "listing_span_days": 36,
      "same_date_pairing": false,
      "attempt_policies": [
        "2+"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "NV-EmbedCode",
          "claude-3-5-sonnet-20241022",
          "deepseek-v3",
          "o3-mini",
          "gpt-4o-2024-05-13",
          "gpt-4-turbo-2024-04-09",
          "Qwen/Qwen2.5-72B-Instruct",
          "meta-llama/Llama-3.1-405B-Instruct",
          "meta-llama/Llama-3.3-70B-Instruct"
        ],
        [
          "NV-EmbedCode",
          "o3",
          "o3-mini",
          "o1",
          "gpt-4.1",
          "gpt-4o-2024-05-13",
          "gpt-4-turbo-2024-04-09",
          "claude-3-7-sonnet-20250219",
          "deepseek-v3",
          "Qwen/Qwen2.5-72B-Instruct",
          "meta-llama/Llama-3.1-405B-Instruct",
          "meta-llama/Llama-3.3-70B-Instruct"
        ]
      ],
      "model_identifiers_consistent": false,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2025-04-10",
          "resolved_percent": 58.2,
          "attempts": "2+",
          "verified": true,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250410_cortexa"
        },
        {
          "entry_date": "2025-05-16",
          "resolved_percent": 68.2,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250516_cortexa_o3"
        }
      ]
    },
    {
      "group_id": "entropo-r2e-qwen3-coder-30b-a3b-instruct",
      "agent": "EntroPO + R2E",
      "model": "Qwen3-Coder-30B-A3B-Instruct",
      "agent_org": "42-b3yond-6ug",
      "n_entries": 2,
      "min_percent": 52.2,
      "max_percent": 60.4,
      "spread_pp": 8.2,
      "listing_span_days": 0,
      "same_date_pairing": true,
      "attempt_policies": [
        "1",
        "2+"
      ],
      "attempt_policy_consistent": false,
      "model_identifiers": [
        [
          "Qwen3-Coder-30B-A3B-Instruct"
        ]
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2025-09-01",
          "resolved_percent": 52.2,
          "attempts": "1",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250901_entroPO_R2E_QwenCoder30BA3B"
        },
        {
          "entry_date": "2025-09-01",
          "resolved_percent": 60.4,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250901_entroPO_R2E_QwenCoder30BA3B_tts"
        }
      ]
    },
    {
      "group_id": "nfactorial-multiple",
      "agent": "nFactorial",
      "model": "Multiple",
      "agent_org": null,
      "n_entries": 2,
      "min_percent": 41.6,
      "max_percent": 49.2,
      "spread_pp": 7.6,
      "listing_span_days": 6,
      "same_date_pairing": false,
      "attempt_policies": [
        "2+"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "claude-3-5-sonnet-20241022",
          "gpt-4o-2024-08-06",
          "o1-preview"
        ],
        [
          "claude-3-5-sonnet-20241022",
          "gpt-4o-2024-08-06"
        ]
      ],
      "model_identifiers_consistent": false,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2024-10-30",
          "resolved_percent": 41.6,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241030_nfactorial"
        },
        {
          "entry_date": "2024-11-05",
          "resolved_percent": 49.2,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241105_nfactorial"
        }
      ]
    },
    {
      "group_id": "solver-undisclosed",
      "agent": "Solver",
      "model": "Undisclosed",
      "agent_org": null,
      "n_entries": 3,
      "min_percent": 43.6,
      "max_percent": 50,
      "spread_pp": 6.4,
      "listing_span_days": 38,
      "same_date_pairing": false,
      "attempt_policies": [
        "2+"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        []
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2024-09-20",
          "resolved_percent": 43.6,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20240920_solver"
        },
        {
          "entry_date": "2024-09-24",
          "resolved_percent": 45.4,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20240924_solver"
        },
        {
          "entry_date": "2024-10-28",
          "resolved_percent": 50,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241028_solver"
        }
      ]
    },
    {
      "group_id": "nfactorial-undisclosed",
      "agent": "nFactorial",
      "model": "Undisclosed",
      "agent_org": null,
      "n_entries": 2,
      "min_percent": 25.8,
      "max_percent": 31.6,
      "spread_pp": 5.8,
      "listing_span_days": 6,
      "same_date_pairing": false,
      "attempt_policies": [
        "2+"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        []
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2024-10-01",
          "resolved_percent": 25.8,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241001_nfactorial"
        },
        {
          "entry_date": "2024-10-07",
          "resolved_percent": 31.6,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241007_nfactorial"
        }
      ]
    },
    {
      "group_id": "warp-multiple",
      "agent": "Warp",
      "model": "Multiple",
      "agent_org": "Warp",
      "n_entries": 2,
      "min_percent": 71,
      "max_percent": 75.6,
      "spread_pp": 4.6,
      "listing_span_days": 70,
      "same_date_pairing": false,
      "attempt_policies": [
        "1",
        "2+"
      ],
      "attempt_policy_consistent": false,
      "model_identifiers": [
        [
          "claude-sonnet-4-20250514",
          "claude-3-7-sonnet-20250219",
          "gemini-2.5-pro",
          "gpt-4.1-2025-04-14"
        ],
        [
          "gpt-5",
          "claude-sonnet-4-20250514"
        ]
      ],
      "model_identifiers_consistent": false,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2025-06-23",
          "resolved_percent": 71,
          "attempts": "1",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250623_warp"
        },
        {
          "entry_date": "2025-09-01",
          "resolved_percent": 75.6,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250901_warp"
        }
      ]
    },
    {
      "group_id": "refact-ai-agent-multiple",
      "agent": "Refact.ai Agent",
      "model": "Multiple",
      "agent_org": "Refact.ai",
      "n_entries": 2,
      "min_percent": 70.4,
      "max_percent": 74.4,
      "spread_pp": 4,
      "listing_span_days": 19,
      "same_date_pairing": false,
      "attempt_policies": [
        "1"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "claude-3-7-sonnet-20250219",
          "o3",
          "o4-mini"
        ],
        [
          "claude-4-sonnet",
          "o4-mini"
        ]
      ],
      "model_identifiers_consistent": false,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2025-05-15",
          "resolved_percent": 70.4,
          "attempts": "1",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250515_Refact_Agent"
        },
        {
          "entry_date": "2025-06-03",
          "resolved_percent": 74.4,
          "attempts": "1",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250603_Refact_Agent_claude-4-sonnet"
        }
      ]
    },
    {
      "group_id": "mini-swe-agent-gpt-5-2",
      "agent": "mini-SWE-agent",
      "model": "GPT 5.2",
      "agent_org": "SWE-agent",
      "n_entries": 3,
      "min_percent": 69,
      "max_percent": 72.8,
      "spread_pp": 3.8,
      "listing_span_days": 68,
      "same_date_pairing": true,
      "attempt_policies": [
        "1"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "gpt-5-2"
        ],
        [
          "gpt-5.2-2025-12-11"
        ]
      ],
      "model_identifiers_consistent": false,
      "scaffold_versions": [
        "1.17.2",
        "2.0.0"
      ],
      "scaffold_version_changed": true,
      "entries": [
        {
          "entry_date": "2025-12-11",
          "resolved_percent": 69,
          "attempts": "1",
          "verified": null,
          "agent_version": "1.17.2",
          "reasoning_effort": null,
          "source_folder": "20251211_mini-v1.17.2_gpt-5.2-2025-12-11"
        },
        {
          "entry_date": "2025-12-11",
          "resolved_percent": 71.8,
          "attempts": "1",
          "verified": null,
          "agent_version": "1.17.2",
          "reasoning_effort": "high",
          "source_folder": "20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high"
        },
        {
          "entry_date": "2026-02-17",
          "resolved_percent": 72.8,
          "attempts": "1",
          "verified": null,
          "agent_version": "2.0.0",
          "reasoning_effort": "high",
          "source_folder": "20260217_mini-v2.0.0_gpt-5-2-high"
        }
      ]
    },
    {
      "group_id": "mini-swe-agent-gpt-5-mini",
      "agent": "mini-SWE-agent",
      "model": "GPT 5 mini",
      "agent_org": "SWE-agent",
      "n_entries": 2,
      "min_percent": 56.2,
      "max_percent": 59.8,
      "spread_pp": 3.6,
      "listing_span_days": 194,
      "same_date_pairing": false,
      "attempt_policies": [
        "1"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "gpt-5-mini-2025-08-07"
        ]
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [
        "1.7.0",
        "2.0.0"
      ],
      "scaffold_version_changed": true,
      "entries": [
        {
          "entry_date": "2025-08-07",
          "resolved_percent": 59.8,
          "attempts": "1",
          "verified": true,
          "agent_version": "1.7.0",
          "reasoning_effort": "medium",
          "source_folder": "20250807_mini-v1.7.0_gpt-5-mini"
        },
        {
          "entry_date": "2026-02-17",
          "resolved_percent": 56.2,
          "attempts": "1",
          "verified": null,
          "agent_version": "2.0.0",
          "reasoning_effort": null,
          "source_folder": "20260217_mini-v2.0.0_gpt-5-mini"
        }
      ]
    },
    {
      "group_id": "epam-ai-run-developer-agent-gpt-4o",
      "agent": "EPAM AI/Run Developer Agent",
      "model": "GPT-4o",
      "agent_org": null,
      "n_entries": 2,
      "min_percent": 24,
      "max_percent": 27,
      "spread_pp": 3,
      "listing_span_days": 57,
      "same_date_pairing": false,
      "attempt_policies": [
        "2+"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "gpt-4o-2024-08-06"
        ]
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2024-08-20",
          "resolved_percent": 24,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20240820_epam-ai-run-gpt-4o"
        },
        {
          "entry_date": "2024-10-16",
          "resolved_percent": 27,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241016_epam-ai-run-gpt-4o"
        }
      ]
    },
    {
      "group_id": "swe-fixer-qwen2-5-7b-72b",
      "agent": "SWE-Fixer",
      "model": "Qwen2.5 (7B + 72B)",
      "agent_org": null,
      "n_entries": 2,
      "min_percent": 30.2,
      "max_percent": 32.8,
      "spread_pp": 2.6,
      "listing_span_days": 98,
      "same_date_pairing": false,
      "attempt_policies": [
        "1",
        "2+"
      ],
      "attempt_policy_consistent": false,
      "model_identifiers": [
        [
          "Qwen 2.5"
        ]
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2024-11-28",
          "resolved_percent": 30.2,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20241128_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor_20241128"
        },
        {
          "entry_date": "2025-03-06",
          "resolved_percent": 32.8,
          "attempts": "1",
          "verified": true,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20250306_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor"
        }
      ]
    },
    {
      "group_id": "mini-swe-agent-claude-4-5-opus",
      "agent": "mini-SWE-agent",
      "model": "Claude 4.5 Opus",
      "agent_org": "SWE-agent",
      "n_entries": 2,
      "min_percent": 74.4,
      "max_percent": 76.8,
      "spread_pp": 2.4,
      "listing_span_days": 85,
      "same_date_pairing": false,
      "attempt_policies": [
        "1"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "claude-4-5-opus"
        ],
        [
          "claude-opus-4-5-20251101"
        ]
      ],
      "model_identifiers_consistent": false,
      "scaffold_versions": [
        "1.16.0",
        "2.0.0"
      ],
      "scaffold_version_changed": true,
      "entries": [
        {
          "entry_date": "2025-11-24",
          "resolved_percent": 74.4,
          "attempts": "1",
          "verified": true,
          "agent_version": "1.16.0",
          "reasoning_effort": "medium",
          "source_folder": "20251124_mini-v1.16.0_claude-opus-4-5-20251101"
        },
        {
          "entry_date": "2026-02-17",
          "resolved_percent": 76.8,
          "attempts": "1",
          "verified": null,
          "agent_version": "2.0.0",
          "reasoning_effort": "high",
          "source_folder": "20260217_mini-v2.0.0_claude-4-5-opus-high"
        }
      ]
    },
    {
      "group_id": "mini-swe-agent-claude-4-5-sonnet",
      "agent": "mini-SWE-agent",
      "model": "Claude 4.5 Sonnet",
      "agent_org": "SWE-agent",
      "n_entries": 2,
      "min_percent": 70.6,
      "max_percent": 71.4,
      "spread_pp": 0.8,
      "listing_span_days": 141,
      "same_date_pairing": false,
      "attempt_policies": [
        "1"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "claude-sonnet-4-5-20250929"
        ]
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [
        "1.13.3",
        "2.0.0"
      ],
      "scaffold_version_changed": true,
      "entries": [
        {
          "entry_date": "2025-09-29",
          "resolved_percent": 70.6,
          "attempts": "1",
          "verified": true,
          "agent_version": "1.13.3",
          "reasoning_effort": null,
          "source_folder": "20250929_mini-v1.13.3_sonnet-4-5-20250929"
        },
        {
          "entry_date": "2026-02-17",
          "resolved_percent": 71.4,
          "attempts": "1",
          "verified": null,
          "agent_version": "2.0.0",
          "reasoning_effort": "high",
          "source_folder": "20260217_mini-v2.0.0_claude-4-5-sonnet-high"
        }
      ]
    },
    {
      "group_id": "salesforce-ai-research-sage-multiple",
      "agent": "Salesforce AI Research SAGE",
      "model": "Multiple",
      "agent_org": "Salesforce AI Research",
      "n_entries": 2,
      "min_percent": 73,
      "max_percent": 73.8,
      "spread_pp": 0.8,
      "listing_span_days": 13,
      "same_date_pairing": false,
      "attempt_policies": [
        "2+"
      ],
      "attempt_policy_consistent": true,
      "model_identifiers": [
        [
          "claude-sonnet-4.5",
          "gpt-5"
        ]
      ],
      "model_identifiers_consistent": true,
      "scaffold_versions": [],
      "scaffold_version_changed": false,
      "entries": [
        {
          "entry_date": "2025-10-21",
          "resolved_percent": 73,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20251021_SalesforceAIResearch_SAGE_bash_only"
        },
        {
          "entry_date": "2025-11-03",
          "resolved_percent": 73.8,
          "attempts": "2+",
          "verified": false,
          "agent_version": null,
          "reasoning_effort": null,
          "source_folder": "20251103_SalesforceAIResearch_SAGE_OpenHands"
        }
      ]
    }
  ]
}