-
Notifications
You must be signed in to change notification settings - Fork 8
Expand file tree
/
Copy pathbenchmark-sync.test.ts
More file actions
131 lines (118 loc) · 3.71 KB
/
Copy pathbenchmark-sync.test.ts
File metadata and controls
131 lines (118 loc) · 3.71 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
import fs from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { type BenchmarkFetch, syncBenchmarks } from '../scripts/fetch/lib/benchmark-sync'
const temporaryDirectories: string[] = []
function leaderboardFetch(results: unknown[]): BenchmarkFetch {
return async () => ({
ok: true,
status: 200,
statusText: 'OK',
async json() {
return {
leaderboards: [{ name: 'Verified', results }],
}
},
})
}
async function createModelRoot(model: Record<string, unknown>): Promise<string> {
const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), 'benchmark-sync-'))
temporaryDirectories.push(rootDir)
await fs.mkdir(path.join(rootDir, 'manifests', 'models'), { recursive: true })
await fs.writeFile(
path.join(rootDir, 'manifests', 'models', `${String(model.id)}.json`),
`${JSON.stringify(model, null, 2)}\n`
)
return rootDir
}
afterEach(async () => {
await Promise.all(
temporaryDirectories
.splice(0)
.map(directory => fs.rm(directory, { force: true, recursive: true }))
)
})
describe('benchmark synchronization', () => {
it('updates an exact SWE-bench result without reformatting the manifest', async () => {
const rootDir = await createModelRoot({
id: 'example',
benchmarks: { sweBench: 40 },
benchmarkTracking: [
{
provider: 'swe-bench',
benchmark: 'sweBench',
leaderboard: 'Verified',
resultId: 'exact-result',
modelLabel: 'Exact model and scaffold',
},
],
description: 'Preserve this value.',
})
const filePath = path.join(rootDir, 'manifests', 'models', 'example.json')
const sourceBeforeSync = await fs.readFile(filePath, 'utf8')
const result = await syncBenchmarks({
rootDir,
write: true,
fetchImpl: leaderboardFetch([
{
folder: 'exact-result',
name: 'Exact model and scaffold',
resolved: 42.5,
},
]),
})
const sourceAfterSync = await fs.readFile(filePath, 'utf8')
expect(result.changes).toHaveLength(1)
expect(sourceAfterSync).toBe(sourceBeforeSync.replace('"sweBench": 40', '"sweBench": 42.5'))
})
it('rejects a changed upstream label instead of fuzzy matching it', async () => {
const rootDir = await createModelRoot({
id: 'example',
benchmarks: { sweBench: 40 },
benchmarkTracking: [
{
provider: 'swe-bench',
benchmark: 'sweBench',
leaderboard: 'Verified',
resultId: 'exact-result',
modelLabel: 'Expected model',
},
],
})
await expect(
syncBenchmarks({
rootDir,
write: true,
fetchImpl: leaderboardFetch([
{ folder: 'exact-result', name: 'Different model', resolved: 42.5 },
]),
})
).rejects.toThrow('label changed')
})
it('does not write when an exact result is missing', async () => {
const rootDir = await createModelRoot({
id: 'example',
benchmarks: { sweBench: 40 },
benchmarkTracking: [
{
provider: 'swe-bench',
benchmark: 'sweBench',
leaderboard: 'Verified',
resultId: 'exact-result',
modelLabel: 'Expected model',
},
],
})
const filePath = path.join(rootDir, 'manifests', 'models', 'example.json')
const sourceBeforeSync = await fs.readFile(filePath, 'utf8')
await expect(
syncBenchmarks({
rootDir,
write: true,
fetchImpl: leaderboardFetch([]),
})
).rejects.toThrow('expected exactly one SWE-bench result')
expect(await fs.readFile(filePath, 'utf8')).toBe(sourceBeforeSync)
})
})