diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 398fd25..1592dfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 24 cache: npm - name: Install dependencies diff --git a/migrations/20260608_add_submissions_benchmark_version_index.sql b/migrations/20260608_add_submissions_benchmark_version_index.sql new file mode 100644 index 0000000..833ffd6 --- /dev/null +++ b/migrations/20260608_add_submissions_benchmark_version_index.sql @@ -0,0 +1,2 @@ +-- Support grouped/count lookups by benchmark version without scanning submissions for each visible version. +CREATE INDEX IF NOT EXISTS idx_submissions_benchmark_version ON submissions(benchmark_version); diff --git a/schema.sql b/schema.sql index 39ef670..85618c6 100644 --- a/schema.sql +++ b/schema.sql @@ -61,6 +61,7 @@ CREATE INDEX IF NOT EXISTS idx_submissions_provider ON submissions(provider); CREATE INDEX IF NOT EXISTS idx_submissions_timestamp ON submissions(timestamp); CREATE INDEX IF NOT EXISTS idx_submissions_score_percentage ON submissions(score_percentage); CREATE INDEX IF NOT EXISTS idx_submissions_official ON submissions(official); +CREATE INDEX IF NOT EXISTS idx_submissions_benchmark_version ON submissions(benchmark_version); CREATE INDEX IF NOT EXISTS idx_tokens_github_username ON tokens(github_username); CREATE INDEX IF NOT EXISTS idx_token_registration_limits_ip ON token_registration_limits(ip); CREATE INDEX IF NOT EXISTS idx_token_registration_limits_created_at ON token_registration_limits(created_at); diff --git a/src/routes/benchmarkVersions.test.ts b/src/routes/benchmarkVersions.test.ts index f9a67fd..34447fe 100644 --- a/src/routes/benchmarkVersions.test.ts +++ b/src/routes/benchmarkVersions.test.ts @@ -1,5 +1,319 @@ import { describe, test, expect } from "vitest"; +import { Hono } from "hono"; +import type { D1Database } from "@cloudflare/workers-types"; +import type { Bindings } from "../types"; import { parseSemver, isValidSemver, compareSemver } from "./benchmarkVersions"; +import { registerBenchmarkVersionRoutes } from "./benchmarkVersions"; + +type BenchmarkVersionFixture = { + id: string; + created_at: string; + current: number; + hidden: number; + semver: string | null; + label: string | null; + release_notes: string | null; + release_url: string | null; +}; + +type SubmissionFixture = { + id: string; + benchmark_version: string | null; +}; + +type QueryLogEntry = { + sql: string; + params: unknown[]; + operation: "all" | "first"; +}; + +function createBenchmarkVersionFixtures(): BenchmarkVersionFixture[] { + return [ + { + id: "v2-release", + created_at: "2026-05-01T00:00:00.000Z", + current: 1, + hidden: 0, + semver: "2.0.0", + label: "2.0.0", + release_notes: "latest stable", + release_url: "https://example.com/v2", + }, + { + id: "v2-rc1", + created_at: "2026-04-15T00:00:00.000Z", + current: 0, + hidden: 0, + semver: "2.0.0-rc.1", + label: null, + release_notes: null, + release_url: null, + }, + { + id: "v1-2-2-dev13", + created_at: "2026-04-01T00:00:00.000Z", + current: 0, + hidden: 0, + semver: "1.2.2-dev.13+gabc1234", + label: null, + release_notes: "dev build", + release_url: null, + }, + { + id: "v1-2-2-dev1", + created_at: "2026-03-20T00:00:00.000Z", + current: 0, + hidden: 0, + semver: "1.2.2-dev.1+g1111111", + label: null, + release_notes: null, + release_url: null, + }, + { + id: "v1-2-1", + created_at: "2026-03-01T00:00:00.000Z", + current: 0, + hidden: 0, + semver: "1.2.1", + label: null, + release_notes: null, + release_url: null, + }, + { + id: "beta10", + created_at: "2026-02-10T00:00:00.000Z", + current: 0, + hidden: 0, + semver: "1.0.0-beta.10", + label: null, + release_notes: null, + release_url: null, + }, + { + id: "beta2", + created_at: "2026-02-02T00:00:00.000Z", + current: 0, + hidden: 0, + semver: "1.0.0-beta.2", + label: null, + release_notes: null, + release_url: null, + }, + { + id: "legacy-high", + created_at: "2026-01-20T00:00:00.000Z", + current: 0, + hidden: 0, + semver: null, + label: "Legacy High", + release_notes: null, + release_url: null, + }, + { + id: "legacy-low", + created_at: "2026-01-01T00:00:00.000Z", + current: 0, + hidden: 0, + semver: "not-semver", + label: null, + release_notes: null, + release_url: null, + }, + { + id: "zero-submissions", + created_at: "2025-12-01T00:00:00.000Z", + current: 0, + hidden: 0, + semver: "0.9.0", + label: null, + release_notes: null, + release_url: null, + }, + { + id: "hidden-with-submissions", + created_at: "2026-06-01T00:00:00.000Z", + current: 0, + hidden: 1, + semver: "9.9.9", + label: "Hidden", + release_notes: null, + release_url: null, + }, + ]; +} + +function createSubmissionFixtures(): SubmissionFixture[] { + const distribution: Record = { + "v2-release": 300, + "v2-rc1": 125, + "v1-2-2-dev13": 180, + "v1-2-2-dev1": 90, + "v1-2-1": 75, + beta10: 60, + beta2: 45, + "legacy-high": 35, + "legacy-low": 20, + "hidden-with-submissions": 70, + }; + + return Object.entries(distribution).flatMap(([benchmarkVersion, count]) => + Array.from({ length: count }, (_, index) => ({ + id: `${benchmarkVersion}-${index}`, + benchmark_version: benchmarkVersion, + })), + ); +} + +function getFixtureLabel(version: BenchmarkVersionFixture): string { + return version.label ?? version.semver ?? version.id.slice(0, 8); +} + +function legacyBenchmarkVersionsSnapshot( + versions: BenchmarkVersionFixture[], + submissions: SubmissionFixture[], +) { + const visibleVersions = versions + .filter((version) => version.hidden === 0) + .sort( + (a, b) => + new Date(b.created_at).getTime() - new Date(a.created_at).getTime(), + ); + + const versionsWithCounts = visibleVersions.map((version) => ({ + id: version.id, + created_at: version.created_at, + is_current: version.current === 1, + submission_count: submissions.filter( + (submission) => submission.benchmark_version === version.id, + ).length, + semver: version.semver ?? null, + label: getFixtureLabel(version), + release_notes: version.release_notes ?? null, + release_url: version.release_url ?? null, + })); + + const withSemver = versionsWithCounts.filter( + (version) => version.semver && isValidSemver(version.semver), + ); + const withoutSemver = versionsWithCounts.filter( + (version) => !version.semver || !isValidSemver(version.semver), + ); + + withSemver.sort((a, b) => compareSemver(a.semver!, b.semver!)); + withoutSemver.sort( + (a, b) => + new Date(b.created_at).getTime() - new Date(a.created_at).getTime(), + ); + + return [...withSemver, ...withoutSemver]; +} + +function createMockD1( + versions: BenchmarkVersionFixture[], + submissions: SubmissionFixture[], +) { + const queryLog: QueryLogEntry[] = []; + + const execute = (sql: string, params: unknown[]) => { + if ( + sql.includes("FROM benchmark_versions") && + sql.includes("WHERE hidden = 0") + ) { + return versions + .filter((version) => version.hidden === 0) + .sort( + (a, b) => + new Date(b.created_at).getTime() - new Date(a.created_at).getTime(), + ); + } + + if ( + sql.includes("FROM benchmark_versions") && + sql.includes("WHERE current = 1 AND hidden = 0") + ) { + return versions + .filter((version) => version.current === 1 && version.hidden === 0) + .sort( + (a, b) => + new Date(b.created_at).getTime() - new Date(a.created_at).getTime(), + ) + .slice(0, 1); + } + + if (sql.includes("FROM submissions") && sql.includes("GROUP BY benchmark_version")) { + const versionIds = new Set(params); + const counts = new Map(); + + for (const submission of submissions) { + if (!submission.benchmark_version || !versionIds.has(submission.benchmark_version)) { + continue; + } + + counts.set( + submission.benchmark_version, + (counts.get(submission.benchmark_version) ?? 0) + 1, + ); + } + + return [...counts.entries()].map(([benchmark_version, count]) => ({ + benchmark_version, + count, + })); + } + + if (sql.includes("FROM submissions") && sql.includes("WHERE benchmark_version = ?")) { + const [benchmarkVersion] = params; + return [ + { + count: submissions.filter( + (submission) => submission.benchmark_version === benchmarkVersion, + ).length, + }, + ]; + } + + throw new Error(`Unexpected query: ${sql}`); + }; + + const db = { + prepare: (sql: string) => { + let boundParams: unknown[] = []; + const statement = { + bind: (...params: unknown[]) => { + boundParams = params; + return statement; + }, + all: async () => { + queryLog.push({ sql, params: boundParams, operation: "all" }); + return { results: execute(sql, boundParams) as T[] }; + }, + first: async () => { + queryLog.push({ sql, params: boundParams, operation: "first" }); + return (execute(sql, boundParams)[0] ?? null) as T | null; + }, + }; + return statement; + }, + } as unknown as D1Database; + + return { db, queryLog }; +} + +async function requestBenchmarkVersions( + db: D1Database, +): Promise<{ versions: unknown[]; generated_at: string }> { + const app = new Hono<{ Bindings: Bindings }>(); + registerBenchmarkVersionRoutes(app); + + const response = await app.request( + "http://localhost/api/benchmark_versions", + {}, + { prod_pinchbench: db }, + ); + + expect(response.status).toBe(200); + return response.json(); +} describe("parseSemver", () => { test("parses basic version", () => { @@ -271,3 +585,46 @@ describe("compareSemver", () => { }); }); }); + +describe("GET /api/benchmark_versions", () => { + test("matches the legacy N+1 count snapshot on a production-equivalent fixture", async () => { + const versions = createBenchmarkVersionFixtures(); + const submissions = createSubmissionFixtures(); + const { db } = createMockD1(versions, submissions); + + const legacySnapshot = legacyBenchmarkVersionsSnapshot(versions, submissions); + const optimizedResponse = await requestBenchmarkVersions(db); + + expect(optimizedResponse.versions).toEqual(legacySnapshot); + expect(optimizedResponse.generated_at).toEqual(expect.any(String)); + expect(new Date(optimizedResponse.generated_at).toString()).not.toBe("Invalid Date"); + }); + + test("uses one grouped submission count query instead of one query per visible version", async () => { + const versions = createBenchmarkVersionFixtures(); + const submissions = createSubmissionFixtures(); + const { db, queryLog } = createMockD1(versions, submissions); + + await requestBenchmarkVersions(db); + + const submissionQueries = queryLog.filter((entry) => + entry.sql.includes("FROM submissions"), + ); + const groupedCountQueries = submissionQueries.filter((entry) => + entry.sql.includes("GROUP BY benchmark_version"), + ); + const legacyPointCountQueries = submissionQueries.filter((entry) => + entry.sql.includes("WHERE benchmark_version = ?"), + ); + + expect(submissionQueries).toHaveLength(1); + expect(groupedCountQueries).toHaveLength(1); + expect(legacyPointCountQueries).toHaveLength(0); + + // The old behavior would issue one submissions count per visible version; keep the API path at two D1 reads total. + expect(queryLog).toHaveLength(2); + expect(groupedCountQueries[0].params).toHaveLength( + versions.filter((version) => version.hidden === 0).length, + ); + }); +}); diff --git a/src/routes/benchmarkVersions.ts b/src/routes/benchmarkVersions.ts index 280f3d7..712e5ae 100644 --- a/src/routes/benchmarkVersions.ts +++ b/src/routes/benchmarkVersions.ts @@ -51,6 +51,11 @@ type BenchmarkVersionRow = { release_url: string | null; }; +type BenchmarkVersionCountRow = { + benchmark_version: string; + count: number; +}; + type BenchmarkVersionResponse = { id: string; created_at: string; @@ -217,26 +222,38 @@ export const registerBenchmarkVersionRoutes = ( ) .all(); - const versionsWithCounts = await Promise.all( - (versions.results ?? []).map(async (version) => { - const countRow = await c.env.prod_pinchbench - .prepare( - "SELECT COUNT(*) as count FROM submissions WHERE benchmark_version = ?", - ) - .bind(version.id) - .first<{ count: number }>(); - return { - id: version.id, - created_at: version.created_at, - is_current: version.current === 1, - submission_count: countRow?.count ?? 0, - semver: version.semver ?? null, - label: getLabel(version), - release_notes: version.release_notes ?? null, - release_url: version.release_url ?? null, - }; - }), - ); + const visibleVersions = versions.results ?? []; + const versionIds = visibleVersions.map((version) => version.id); + const submissionCounts = new Map(); + + if (versionIds.length > 0) { + const placeholders = versionIds.map(() => "?").join(", "); + const countRows = await c.env.prod_pinchbench + .prepare( + `SELECT benchmark_version, COUNT(*) as count + FROM submissions + WHERE benchmark_version IN (${placeholders}) + GROUP BY benchmark_version`, + ) + .bind(...versionIds) + .all(); + + // Count all visible versions in one indexed aggregate instead of scanning submissions once per version. + for (const row of countRows.results ?? []) { + submissionCounts.set(row.benchmark_version, row.count); + } + } + + const versionsWithCounts = visibleVersions.map((version) => ({ + id: version.id, + created_at: version.created_at, + is_current: version.current === 1, + submission_count: submissionCounts.get(version.id) ?? 0, + semver: version.semver ?? null, + label: getLabel(version), + release_notes: version.release_notes ?? null, + release_url: version.release_url ?? null, + })); const sortedVersions = sortVersions(versionsWithCounts);