stage-pilot/docs/benchmarks/stagepilot-latest.json at main · KIM3310/stage-pilot · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
{
  "caseCount": 40,
  "generatedAt": "2026-03-24T06:16:41.757Z",
  "improvements": {
    "loopVsBaseline": 65,
    "loopVsMiddleware": 25,
    "middlewareVsBaseline": 40
  },
  "seed": 20260228,
  "strategies": [
    {
      "avgAttemptsUsed": 1,
      "avgLatencyMs": 0.02,
      "failedCaseIds": [
        "bench-2",
        "bench-3",
        "bench-4",
        "bench-5",
        "bench-6",
        "bench-8",
        "bench-10",
        "bench-11",
        "bench-13",
        "bench-14",
        "bench-16",
        "bench-17",
        "bench-18",
        "bench-19",
        "bench-20",
        "bench-22",
        "bench-23",
        "bench-24",
        "bench-25",
        "bench-26",
        "bench-28",
        "bench-30",
        "bench-31",
        "bench-33",
        "bench-34",
        "bench-36",
        "bench-37",
        "bench-38",
        "bench-39",
        "bench-40"
      ],
      "p95LatencyMs": 0.05,
      "parseSuccessCount": 10,
      "planSuccessCount": 10,
      "strategy": "baseline",
      "successRate": 25,
      "totalCases": 40
    },
    {
      "avgAttemptsUsed": 1,
      "avgLatencyMs": 0.13,
      "failedCaseIds": [
        "bench-4",
        "bench-8",
        "bench-10",
        "bench-14",
        "bench-17",
        "bench-18",
        "bench-19",
        "bench-24",
        "bench-28",
        "bench-30",
        "bench-34",
        "bench-37",
        "bench-38",
        "bench-39"
      ],
      "p95LatencyMs": 0.39,
      "parseSuccessCount": 26,
      "planSuccessCount": 26,
      "strategy": "middleware",
      "successRate": 65,
      "totalCases": 40
    },
    {
      "avgAttemptsUsed": 1.35,
      "avgLatencyMs": 0.06,
      "failedCaseIds": ["bench-14", "bench-17", "bench-34", "bench-37"],
      "p95LatencyMs": 0.1,
      "parseSuccessCount": 36,
      "planSuccessCount": 36,
      "strategy": "middleware+ralph-loop",
      "successRate": 90,
      "totalCases": 40
    }
  ]
}