{
  "dataset": "yylo-repository-benchmark-cases",
  "schema_version": "yylo_seo_benchmark_case_library.v1",
  "dataset_version": 1,
  "description": "Repository-specific coding-agent benchmark cases mined from the public git histories of the released YYLO ecosystem repositories: each case pins one real base commit, the real later grading commit that carries its deterministic checks, a candidate-visible task, environment setup notes, and the exact grading command, with both fail and pass states verified on the stated date.",
  "publisher": "yylo.dev",
  "canonical_url": "https://yylo.dev/guides/repository-benchmark-cases",
  "download_url": "https://yylo.dev/data/yylo-repository-benchmark-cases-v1.json",
  "source": {
    "method": "Mined and verified from the public git histories of the three released YYLO ecosystem repositories. A case enters the library only when a later public commit both resolves a stated engineering task and adds or tightens deterministic offline checks for it, and when the checks run standalone from a plain clone of the repository (suites bound to a monorepo or controller layout are excluded).",
    "mined_at": "2026-08-29",
    "repositories": [
      {
        "id": "yylo-ledger",
        "clone_url": "https://github.com/yylo-dev/yylo-ledger.git"
      },
      {
        "id": "yylo",
        "clone_url": "https://github.com/yylo-dev/yylo.git"
      },
      {
        "id": "yylo-benchmark",
        "clone_url": "https://github.com/yylo-dev/yylo-benchmark.git"
      }
    ]
  },
  "population": {
    "cases": 6,
    "repositories": 3,
    "languages": [
      "python",
      "typescript"
    ],
    "categories": [
      "backend",
      "release",
      "harness",
      "evaluation"
    ],
    "order": "by case_id ascending"
  },
  "field_notes": {
    "case_id": "stable library identifier for the case; never reused across dataset versions",
    "repository_id": "identifier of the public repository the case is pinned to; resolves through source.repositories",
    "category": "library comparison vocabulary for the kind of engineering work (backend, release, harness, evaluation)",
    "language": "primary implementation language of the change surface",
    "base_commit": "full commit hash of the exact tree a candidate starts from; the parent of the grading commit",
    "grading_commit": "full commit hash of the real later public fix whose test changes form the grading patch; evaluator-side material, never shown to the candidate",
    "task": "the candidate-visible engineering task, stated as behavior; it names no implementation site",
    "environment.setup": "shell commands that materialize the baseline from a plain clone",
    "environment.notes": "interpreter requirements, dependencies, and offline/runtime characteristics of the graded suite",
    "grading.test_files": "the files whose grading-commit diff constitutes the grading patch",
    "grading.apply": "command that applies exactly the grading patch onto the candidate tree",
    "grading.command": "the check command; the case resolves when it exits zero with every test passing",
    "grading.resolved_if": "the resolution condition; a tree passing it need not match the grading commit's implementation",
    "verification.base_with_grading_patch": "observed result of the grading command on the untouched base commit with only the grading patch applied",
    "verification.grading_commit_tree": "observed result of the grading command on the grading commit's own tree",
    "provenance": "where the case was mined from, the grading commit's subject line, and its authoring date"
  },
  "rights_note": "The pinned commits and their tests belong to the yylo-dev public repositories under their licenses; this library wraps those histories for reuse with attribution to the repositories it names. It is not a YYLO product measurement, and no figure in it describes the performance of any model or agent.",
  "verification_environment": {
    "date": "2026-08-29",
    "node": "v26.2.0",
    "python": "3.13.15",
    "pytest": "9.1.1",
    "vitest": "1.6.1 in the yylo repository; 1.6.1 resolved from the ^1.0.4 pin in yylo-benchmark's lockfile",
    "network": "Dependency installation was the only network use. Every graded suite was re-run on 2026-08-29 under a macOS sandbox-exec profile denying all networking and passed unchanged: 7, 6, 6, and 25 pytest tests; 222 and 18 vitest tests."
  },
  "cases_sha256": "sha256:bf7ead5c363b87e2fcc34b08e64bac74f0b5105ba2e1d3c744a4173e5f3c3669",
  "cases": [
    {
      "case_id": "benchmark-boundary-block-scalars",
      "repository_id": "yylo-benchmark",
      "category": "evaluation",
      "language": "typescript",
      "base_commit": "a8a795ac7de71a0cdc3f4ccf39d8aab7f60a702c",
      "grading_commit": "2b32dde6d5c28a6481422cc8ddcfcead244fb690",
      "task": "The reviewed workflow boundary consumes a strict YAML subset and owns durable dispatch intent for tracked workflow experiments. Two requirements. First, its block-scalar reader must accept every multiline prompt form the benchmark planner's canonical emitter produces: blank interior lines survive as content, continuation lines indented deeper than the auto-detected block indent round-trip, chomping (clip, strip, keep) is byte-exact including trailing blank lines and spaces, explicit indentation indicators such as |2, |2-, and |-2 are accepted, and tabs stay legal inside block-scalar content while remaining rejected in structural positions. A canonical multiline prompt must round-trip byte-exactly through the exact dispatched argv. Second, the boundary must fully validate its compiled workflow bytes - parse them, resolve the exact substituted command, and check the deterministic policy prefix - before recording any durable dispatch intent; a rejected request must leave no journal entry and must later reconcile as proven_not_dispatched.",
      "environment": {
        "setup": [
          "git clone https://github.com/yylo-dev/yylo-benchmark.git",
          "cd yylo-benchmark",
          "git checkout a8a795ac7de71a0cdc3f4ccf39d8aab7f60a702c",
          "npm ci"
        ],
        "notes": "Node >=20.10.0 (verified on v26.2.0). The graded file executes the real reviewed boundary module as a local node subprocess; there is no provider dispatch, and the run is offline after dependency installation."
      },
      "grading": {
        "test_files": [
          "test/boundary/workflow-boundary-module.test.ts"
        ],
        "apply": "git diff a8a795ac7de71a0cdc3f4ccf39d8aab7f60a702c 2b32dde6d5c28a6481422cc8ddcfcead244fb690 -- test/boundary/workflow-boundary-module.test.ts | git apply",
        "command": "npx vitest run test/boundary/workflow-boundary-module.test.ts",
        "resolved_if": "every test in the file passes"
      },
      "verification": {
        "date": "2026-08-29",
        "base_with_grading_patch": "12 failed, 6 passed (the applied grading expectations cover round-trip of canonical block-scalar forms through the dispatched argv and rejection of unparsable compiled workflow bytes before any durable intent)",
        "grading_commit_tree": "18 passed, 18 total"
      },
      "provenance": {
        "mined_from": "public git history of yylo-dev/yylo-benchmark",
        "clone_url": "https://github.com/yylo-dev/yylo-benchmark.git",
        "fix_subject": "fix(benchmark): accept canonical multiline block scalars and validate compiled workflow before durable intent [D0tTNr]",
        "fix_authored": "2026-08-25"
      }
    },
    {
      "case_id": "ledger-archived-id-reservation",
      "repository_id": "yylo-ledger",
      "category": "backend",
      "language": "python",
      "base_commit": "71fbfe4ad0fb0472d3d88f9847d1d77a3d42dcd1",
      "grading_commit": "0cf9fe0199e8a7eb595e4849127f3ff2046ec856",
      "task": "Archived record IDs are reserved forever. Importing any record whose ID already exists in the immutable archive must be refused before any other validation runs and before any bytes are written: no task path may appear, no history event may be appended, and the query cache must stay byte-identical. Non-archived imports keep their existing validation order, including the reserved-creation-context rule that refuses ledger-owned creation context on imported records.",
      "environment": {
        "setup": [
          "git clone https://github.com/yylo-dev/yylo-ledger.git",
          "cd yylo-ledger",
          "git checkout 71fbfe4ad0fb0472d3d88f9847d1d77a3d42dcd1",
          "python -m venv .venv && . .venv/bin/activate",
          "pip install pytest ruamel.yaml"
        ],
        "notes": "Python >=3.8 (verified on 3.13.15). Runtime dependency: ruamel.yaml. The graded integration suite drives a real git binary through the storage layer and needs roughly thirty seconds; it is offline."
      },
      "grading": {
        "test_files": [
          "tests/integration/test_archive_query_cache.py"
        ],
        "apply": "git diff 71fbfe4ad0fb0472d3d88f9847d1d77a3d42dcd1 0cf9fe0199e8a7eb595e4849127f3ff2046ec856 -- tests/integration/test_archive_query_cache.py | git apply",
        "command": "PYTHONPATH=src python -m pytest tests/integration/test_archive_query_cache.py -q",
        "resolved_if": "every test in the file passes"
      },
      "verification": {
        "date": "2026-08-29",
        "base_with_grading_patch": "1 failed, 24 passed (the new reservation test observes a partial write for one archived-ID import path)",
        "grading_commit_tree": "25 passed, 25 total"
      },
      "provenance": {
        "mined_from": "public git history of yylo-dev/yylo-ledger",
        "clone_url": "https://github.com/yylo-dev/yylo-ledger.git",
        "fix_subject": "fix(storage): reject archived record replacements first [N1ctPR]",
        "fix_authored": "2026-08-28"
      }
    },
    {
      "case_id": "ledger-artifact-revision-creation-context",
      "repository_id": "yylo-ledger",
      "category": "backend",
      "language": "python",
      "base_commit": "d78903c79a1cb3f0664815079f6e7e15bb4fb990",
      "grading_commit": "3f1457ae6068766e4e2334209c0450e53bc7f4e0",
      "task": "Every record stores its Git creation context - the repositories, roles, head SHAs, refs, and dirty flags observed when it was created - in system metadata. Revising an artifact record currently drops that creation context from the revision's manifest, so provenance silently disappears after the first revision. Make revisions carry the original creation context forward unchanged and keep the revised record valid under the record schema.",
      "environment": {
        "setup": [
          "git clone https://github.com/yylo-dev/yylo-ledger.git",
          "cd yylo-ledger",
          "git checkout d78903c79a1cb3f0664815079f6e7e15bb4fb990",
          "python -m venv .venv && . .venv/bin/activate",
          "pip install pytest ruamel.yaml"
        ],
        "notes": "Python >=3.8 (verified on 3.13.15). Runtime dependency: ruamel.yaml. The graded unit suite is offline and finishes in a few seconds."
      },
      "grading": {
        "test_files": [
          "tests/unit/test_git_creation_context.py"
        ],
        "apply": "git diff d78903c79a1cb3f0664815079f6e7e15bb4fb990 3f1457ae6068766e4e2334209c0450e53bc7f4e0 -- tests/unit/test_git_creation_context.py | git apply",
        "command": "PYTHONPATH=src python -m pytest tests/unit/test_git_creation_context.py -q",
        "resolved_if": "every test in the file passes"
      },
      "verification": {
        "date": "2026-08-29",
        "base_with_grading_patch": "1 failed, 5 passed (the new revision test observes KeyError: 'creation_context' on the revised manifest)",
        "grading_commit_tree": "6 passed, 6 total"
      },
      "provenance": {
        "mined_from": "public git history of yylo-dev/yylo-ledger",
        "clone_url": "https://github.com/yylo-dev/yylo-ledger.git",
        "fix_subject": "fix(artifacts): preserve creation context across revisions [RPV7Gb]",
        "fix_authored": "2026-08-28"
      }
    },
    {
      "case_id": "ledger-release-version-bumping",
      "repository_id": "yylo-ledger",
      "category": "release",
      "language": "python",
      "base_commit": "b07ae587291f26249cfab419b102f34ca850de25",
      "grading_commit": "60d9aaac8c96d72b5c322be6a2685cd1ea2d5e04",
      "task": "The release version-bump helper must operate on the canonical package identity. It still reads and writes the retired src/kanban/__init__.py layout, and its strict three-part parser rejects legitimate PEP 440 prerelease versions such as 0.2.1rc1. Make it parse and write the canonical src/yylo_ledger/__init__.py identity, accept final and explicit a, b, and rc versions, and keep ordinary tuple comparison PEP 440-correct so a prerelease never sorts above the final release it precedes. The helper stays a local, dry-run-capable script; publishing is out of scope, and network lookups it performs must remain mocked in tests.",
      "environment": {
        "setup": [
          "git clone https://github.com/yylo-dev/yylo-ledger.git",
          "cd yylo-ledger",
          "git checkout b07ae587291f26249cfab419b102f34ca850de25",
          "python -m venv .venv && . .venv/bin/activate",
          "pip install pytest ruamel.yaml"
        ],
        "notes": "Python >=3.8 (verified on 3.13.15). Runtime dependency: ruamel.yaml. The graded unit suite is hermetic: PyPI reachability and published-version lookups are mocked, and the run is offline."
      },
      "grading": {
        "test_files": [
          "tests/unit/test_bump_version.py"
        ],
        "apply": "git diff b07ae587291f26249cfab419b102f34ca850de25 60d9aaac8c96d72b5c322be6a2685cd1ea2d5e04 -- tests/unit/test_bump_version.py | git apply",
        "command": "PYTHONPATH=src python -m pytest tests/unit/test_bump_version.py -q",
        "resolved_if": "every test in the file passes"
      },
      "verification": {
        "date": "2026-08-29",
        "base_with_grading_patch": "6 failed, 6 total (canonical-identity, prerelease-parse, ordering, and mocked-lookup expectations all fail against the retired layout and strict parser)",
        "grading_commit_tree": "6 passed, 6 total"
      },
      "provenance": {
        "mined_from": "public git history of yylo-dev/yylo-ledger",
        "clone_url": "https://github.com/yylo-dev/yylo-ledger.git",
        "fix_subject": "fix(release): repair canonical Ledger version bumping [2ALVMc]",
        "fix_authored": "2026-08-29"
      }
    },
    {
      "case_id": "ledger-search-creation-git-dirty-filter",
      "repository_id": "yylo-ledger",
      "category": "backend",
      "language": "python",
      "base_commit": "d78903c79a1cb3f0664815079f6e7e15bb4fb990",
      "grading_commit": "249689d816a838bf3c64f1ea53fb703bcd837062",
      "task": "The record search API accepts creation-Git filters - role, head_sha, ref, worktree_dirty, and repository_id - over the indexed creation context. Filtering by worktree_dirty, for either true or false, currently raises a query-time database error instead of returning results. Make worktree_dirty filters return exactly the records whose creation context carries that flag, return empty results for the opposite value, and leave the other creation-Git filters working as before.",
      "environment": {
        "setup": [
          "git clone https://github.com/yylo-dev/yylo-ledger.git",
          "cd yylo-ledger",
          "git checkout d78903c79a1cb3f0664815079f6e7e15bb4fb990",
          "python -m venv .venv && . .venv/bin/activate",
          "pip install pytest ruamel.yaml"
        ],
        "notes": "Python >=3.8 (verified on 3.13.15). Runtime dependency: ruamel.yaml. The graded unit suite is offline and finishes in a few seconds."
      },
      "grading": {
        "test_files": [
          "tests/unit/test_record_search.py"
        ],
        "apply": "git diff d78903c79a1cb3f0664815079f6e7e15bb4fb990 249689d816a838bf3c64f1ea53fb703bcd837062 -- tests/unit/test_record_search.py | git apply",
        "command": "PYTHONPATH=src python -m pytest tests/unit/test_record_search.py -q",
        "resolved_if": "every test in the file passes"
      },
      "verification": {
        "date": "2026-08-29",
        "base_with_grading_patch": "2 failed, 5 passed (both parametrized cases of the new test fail with sqlite3.OperationalError: no such column: g.worktree_dirty)",
        "grading_commit_tree": "7 passed, 7 total"
      },
      "provenance": {
        "mined_from": "public git history of yylo-dev/yylo-ledger",
        "clone_url": "https://github.com/yylo-dev/yylo-ledger.git",
        "fix_subject": "fix(records): map dirty search to canonical column [TOsv00]",
        "fix_authored": "2026-08-28"
      }
    },
    {
      "case_id": "yylo-project-model-shortcuts",
      "repository_id": "yylo",
      "category": "harness",
      "language": "typescript",
      "base_commit": "c8fd58ce625235130341f07b6f9952edcf5c0deb",
      "grading_commit": "7fd4e9c1f9df5d8e37d059a85456a181ceebe700",
      "task": "yylo resolves models in this order: CLI --model, then the configured subagent default, then the built-in default. Projects need per-subagent model shortcuts in .juno_task/config.json: a modelShortcuts map keyed by subagent (claude, cursor, codex, gemini, pi) whose entries start with a colon and whose targets are opaque non-empty model strings that may reference other project shortcuts or shipped shortcuts. Lookup is scoped to the selected subagent; a project entry overrides the same shipped shortcut while every other shipped shortcut remains available; shortcut chains resolve recursively; and cycles, unknown shortcuts, and malformed shortcut data fail with an actionable error instead of dispatching a different model. A configured subagent default may itself be a shortcut, the resolved identity must reach the dispatched service, and cursor reads its own cursor shortcut map even though it uses the Claude service wrapper.",
      "environment": {
        "setup": [
          "git clone https://github.com/yylo-dev/yylo.git",
          "cd yylo",
          "git checkout c8fd58ce625235130341f07b6f9952edcf5c0deb",
          "npm ci"
        ],
        "notes": "Node >=20.10.0 (verified on v26.2.0). npm ci then vitest; the graded files cover configuration, engine, subagent-model, and shell-backend units with no provider calls, offline after dependency installation."
      },
      "grading": {
        "test_files": [
          "src/core/__tests__/config.test.ts",
          "src/core/__tests__/engine.test.ts",
          "src/core/__tests__/subagent-models.test.ts",
          "src/core/__tests__/shell-backend-structured-output.test.ts"
        ],
        "apply": "git diff c8fd58ce625235130341f07b6f9952edcf5c0deb 7fd4e9c1f9df5d8e37d059a85456a181ceebe700 -- src/core/__tests__/config.test.ts src/core/__tests__/engine.test.ts src/core/__tests__/subagent-models.test.ts src/core/__tests__/shell-backend-structured-output.test.ts | git apply",
        "command": "npx vitest run src/core/__tests__/config.test.ts src/core/__tests__/engine.test.ts src/core/__tests__/subagent-models.test.ts src/core/__tests__/shell-backend-structured-output.test.ts",
        "resolved_if": "every test in all four files passes"
      },
      "verification": {
        "date": "2026-08-29",
        "base_with_grading_patch": "4 failed (exactly one per file: schema-with-all-valid-configurations; propagate project shortcuts and selected subagent to the service environment; Claude service wrapper selection for cursor; opaque project shortcut as a configured subagent default)",
        "grading_commit_tree": "222 passed across the four files"
      },
      "provenance": {
        "mined_from": "public git history of yylo-dev/yylo",
        "clone_url": "https://github.com/yylo-dev/yylo.git",
        "fix_subject": "feat(models): restore project shortcut resolution",
        "fix_authored": "2026-08-28"
      }
    }
  ]
}