From 7fd67a45252f96c82387b69d8441c87890698c10 Mon Sep 17 00:00:00 2001 From: Devdatta Talele Date: Thu, 11 Jun 2026 17:38:22 +0530 Subject: [PATCH] Add slates doctor command --- packages/cli/README.md | 135 ++++++++ packages/cli/src/cli.ts | 39 ++- packages/cli/src/commands/doctor-checks.ts | 187 +++++++++++ packages/cli/src/commands/doctor.test.ts | 341 +++++++++++++++++++++ packages/cli/src/commands/doctor.ts | 209 +++++++++++++ packages/cli/src/commands/index.ts | 1 + packages/cli/src/lib/integration.ts | 2 +- 7 files changed, 908 insertions(+), 6 deletions(-) create mode 100644 packages/cli/README.md create mode 100644 packages/cli/src/commands/doctor-checks.ts create mode 100644 packages/cli/src/commands/doctor.test.ts create mode 100644 packages/cli/src/commands/doctor.ts diff --git a/packages/cli/README.md b/packages/cli/README.md new file mode 100644 index 0000000000..2ba6cb95ab --- /dev/null +++ b/packages/cli/README.md @@ -0,0 +1,135 @@ +# @slates/cli + +Reference CLI for the Metorial integrations workspace. Provides per-integration +and global commands for setup, auth, tool invocation, profile management, and +workspace-wide audits. + +## Install + +The CLI runs through Bun directly from the workspace: + +```bash +bun packages/cli/src/cli.ts +``` + +Or via the root-level alias: + +```bash +bun run integrations:cli +``` + +## Global commands + +### `slates test` + +Run vitest across every integration in the workspace. + +```bash +bun run integrations:cli test +bun run integrations:cli test -- --reporter=verbose +``` + +Arguments after `--` are forwarded to vitest. + +### `slates doctor` + +Workspace-wide consistency audit. Walks every `integrations/` folder and +reports gaps against the standards used in the canonical Google-family +integrations (`gmail`, `google-meet`, `youtube`, `google-analytics`). + +```bash +bun run integrations:cli doctor +bun run integrations:cli doctor --check=raw-throws --all +bun run integrations:cli doctor --integration=linear +bun run integrations:cli doctor --json > doctor.json +``` + +#### Checks + +| Check | Severity | What it catches | +|-------|----------|------------------| +| `raw-throws` | `error` | `throw new Error(...)` outside test files. Tags partial migrations where `src/lib/errors.ts` already exists. | +| `scope-file` | `error` | Integration has `src/auth.ts` containing OAuth logic but no `src/scopes.ts`. | +| `vitest-config` | `error` | Test files present but no `vitest.config.ts`. | +| `contract-tests` | `warn` | `src/auth.ts` present but no `*.contract.test.ts`. | +| `zod-describe` | `warn` | Top-level Zod field declarations without `.describe()`. Multi-line chains handled. | +| `readme` | `info` | Integration has no `README.md`. | + +Severity bucketing: + +- `error` — breaks `@lowerdeck/error` tagging, runtime behavior, or test discovery. +- `warn` — degrades AI tool-calling quality, test coverage, or platform integration. +- `info` — best-practice gap, usually documentation or metadata. + +#### Flags + +| Flag | Effect | +|------|--------| +| `--check=` | Run only the named check. Drills into the per-integration breakdown. | +| `--integration=` | Limit the audit to a single integration directory. | +| `--all` | Show every failing integration (default: top 10 per check). | +| `--json` | Emit a machine-readable report instead of the pretty table. | +| `--no-color` | Disable ANSI color in pretty output (default: auto-detected from TTY). | +| `--include-test-integrations` | Include `test-integrations/` in the audit. | + +#### Example output + +``` +Slates Doctor - 1121 integrations audited + +ERR raw-throws 584 src/ has raw `throw new Error(...)` - use lib/errors.ts helpers +ERR scope-file 251 OAuth integration without src/scopes.ts + ok vitest-config 0 test files present but no vitest.config.ts +WRN contract-tests 1092 src/auth.ts present but no *.contract.test.ts +WRN zod-describe 1119 Zod field declarations without .describe() + ok readme 0 integration is missing a README.md + +Total: 3046 findings (835 error, 2211 warn, 0 info) +``` + +#### JSON shape + +```jsonc +{ + "auditedIntegrations": 1121, + "totalFailures": 3176, + "totalsBySeverity": { "error": 630, "warn": 1296, "info": 1230 }, + "checks": [ + { + "name": "raw-throws", + "severity": "error", + "description": "src/ has raw `throw new Error(...)` - use lib/errors.ts helpers", + "failures": [ + { "integration": "optimizely", "detail": "69 raw throws (no lib/errors.ts)" } + ] + } + ] +} +``` + +#### Relationship to `scripts/validate-pr-integrations.ts` + +`validate-pr-integrations.ts` validates **schema diffs for paths changed in a +PR** by building both base and head, capturing snapshots, and comparing +provider / tool / auth-method schemas. It is the per-PR safety gate. + +`slates doctor` complements it by auditing **code-shape consistency across the +whole workspace** without any build step. Use both: doctor for periodic health +snapshots and standards tracking, `validate-pr-integrations` for per-PR +behavioral safety. + +## Per-integration commands + +Invoked as `slates `, where `` is the name +of a directory under `integrations/` or `test-integrations/`. + +- `slates setup` — interactive integration setup +- `slates profiles {add,list,get,use,remove}` — profile management +- `slates tools {list,get,schema,call}` — invoke tools +- `slates auth {list,get,setup,refresh,credentials}` — manage auth state +- `slates config {get,set,schema}` — manage runtime config +- `slates test` — run vitest against this integration with a profile context +- `slates repl` — open an interactive REPL with the loaded provider + +Run `bun run integrations:cli --help` for the full per-command +flag list. diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 35f8418b24..4a2e260530 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -17,6 +17,7 @@ import { refreshAuth, removeProfile, runAllIntegrationTests, + runDoctor, runVitestWithProfile, setConfig, setupAuth, @@ -39,15 +40,18 @@ let printResult = async (cb: () => Promise) => { let cli = sade('slates'); let argv = process.argv.slice(2); -let isGlobalTestCommand = argv[0] === 'test'; -let integration = isGlobalTestCommand ? null : argv[0]; +let GLOBAL_COMMANDS = new Set(['test', 'doctor']); +let isGlobalCommand = GLOBAL_COMMANDS.has(argv[0] ?? ''); +let integration = isGlobalCommand ? null : argv[0]; -if (!isGlobalTestCommand && (!integration || integration.startsWith('-'))) { - console.error('Usage: slates \n slates test'); +if (!isGlobalCommand && (!integration || integration.startsWith('-'))) { + console.error( + 'Usage: slates \n slates test\n slates doctor' + ); process.exit(1); } -if (isGlobalTestCommand) { +if (isGlobalCommand) { cli.command('test').action(() => printResult(async () => { let separatorIndex = process.argv.indexOf('--'); @@ -57,6 +61,31 @@ if (isGlobalTestCommand) { }) ); + cli + .command('doctor') + .describe('Audit every integration in the workspace for consistency gaps.') + .option('--check', 'Run a single named check (see README for the full list)') + .option('--integration', 'Limit the audit to one integration by directory name') + .option('--json', 'Emit a machine-readable JSON report instead of the pretty table') + .option('--all', 'Show every failing integration instead of only the top 10') + .option('--no-color', 'Disable ANSI color in pretty output (auto-detected from TTY)') + .option( + '--include-test-integrations', + 'Include integrations under test-integrations/ in the audit' + ) + .action(opts => + printResult(() => + runDoctor({ + check: opts.check, + integration: opts.integration, + json: Boolean(opts.json), + all: Boolean(opts.all), + noColor: opts.color === false, + includeTestIntegrations: Boolean(opts['include-test-integrations']) + }) + ) + ); + cli.parse([process.argv[0] ?? 'bun', process.argv[1] ?? 'slates', ...argv]); } else { cli diff --git a/packages/cli/src/commands/doctor-checks.ts b/packages/cli/src/commands/doctor-checks.ts new file mode 100644 index 0000000000..ce9dc6dd76 --- /dev/null +++ b/packages/cli/src/commands/doctor-checks.ts @@ -0,0 +1,187 @@ +import { readdir, readFile } from 'fs/promises'; +import path from 'path'; +import { pathExists, type WorkspaceIntegrationSummary } from '../lib/integration'; + +export type Severity = 'error' | 'warn' | 'info'; + +export interface CheckContext { + integration: WorkspaceIntegrationSummary; + srcFiles: string[]; + hasFile: (relative: string) => Promise; + read: (relative: string) => Promise; + readJson: >(relative: string) => Promise; +} + +export interface CheckResult { + failed: boolean; + detail: string; +} + +export interface CheckSpec { + name: string; + severity: Severity; + description: string; + run: (ctx: CheckContext) => Promise; +} + +export let SEVERITY_ORDER: Record = { + error: 0, + warn: 1, + info: 2 +}; + +let SKIP_DIRS = new Set(['node_modules', 'dist', '.turbo']); +let ZOD_FIELD_REGEX = /^\s+\w+:\s*z\.\w+\(/; +let RAW_THROW_REGEX = /throw new Error\(/g; +let OAUTH_PROBE_REGEX = /oauth/i; + +let walkTsFiles = async (dir: string): Promise => { + if (!(await pathExists(dir))) return []; + let collected: string[] = []; + let stack = [dir]; + while (stack.length > 0) { + let current = stack.pop()!; + let entries = await readdir(current, { withFileTypes: true }); + for (let entry of entries) { + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + stack.push(path.join(current, entry.name)); + } else if ( + entry.isFile() && + (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx')) + ) { + collected.push(path.join(current, entry.name)); + } + } + } + return collected; +}; + +let collapseChains = (source: string) => source.replace(/\n\s*\./g, '.'); + +let countRawThrows = async (srcFiles: string[], read: (file: string) => Promise) => { + let count = 0; + await Promise.all( + srcFiles.map(async file => { + if (file.endsWith('.test.ts')) return; + let content = await read(file); + let matches = content.match(RAW_THROW_REGEX); + if (matches) count += matches.length; + }) + ); + return count; +}; + +let countMissingZodDescribes = async ( + srcFiles: string[], + read: (file: string) => Promise +) => { + let missing = 0; + let scanned = 0; + await Promise.all( + srcFiles.map(async file => { + if (file.endsWith('.test.ts')) return; + let content = await read(file); + let collapsed = collapseChains(content); + for (let line of collapsed.split('\n')) { + if (!ZOD_FIELD_REGEX.test(line)) continue; + scanned++; + if (!line.includes('.describe(')) missing++; + } + }) + ); + return { missing, scanned }; +}; + +export let CHECKS: CheckSpec[] = [ + { + name: 'raw-throws', + severity: 'error', + description: 'src/ has raw `throw new Error(...)` - use lib/errors.ts helpers', + run: async ({ srcFiles, hasFile, read }) => { + let throwCount = await countRawThrows(srcFiles, read); + if (throwCount === 0) return { failed: false, detail: '' }; + let hasErrorsHelper = await hasFile('src/lib/errors.ts'); + let suffix = hasErrorsHelper + ? ' (partial: lib/errors.ts exists)' + : ' (no lib/errors.ts)'; + return { + failed: true, + detail: `${throwCount} raw throw${throwCount === 1 ? '' : 's'}${suffix}` + }; + } + }, + { + name: 'scope-file', + severity: 'error', + description: 'OAuth integration without src/scopes.ts', + run: async ({ hasFile, read }) => { + if (!(await hasFile('src/auth.ts'))) return null; + let authContent = await read('src/auth.ts'); + if (!OAUTH_PROBE_REGEX.test(authContent)) return null; + if (await hasFile('src/scopes.ts')) return { failed: false, detail: '' }; + return { failed: true, detail: 'OAuth detected in auth.ts but no scopes.ts' }; + } + }, + { + name: 'vitest-config', + severity: 'error', + description: 'test files present but no vitest.config.ts', + run: async ({ srcFiles, hasFile }) => { + let hasTests = srcFiles.some(file => file.endsWith('.test.ts')); + if (!hasTests) return null; + if (await hasFile('vitest.config.ts')) return { failed: false, detail: '' }; + return { failed: true, detail: 'tests present, no vitest.config.ts' }; + } + }, + { + name: 'contract-tests', + severity: 'warn', + description: 'src/auth.ts present but no *.contract.test.ts', + run: async ({ srcFiles, hasFile }) => { + if (!(await hasFile('src/auth.ts'))) return null; + let hasContractTest = srcFiles.some(file => file.endsWith('.contract.test.ts')); + if (hasContractTest) return { failed: false, detail: '' }; + return { failed: true, detail: 'auth.ts present, no contract tests' }; + } + }, + { + name: 'zod-describe', + severity: 'warn', + description: 'Zod field declarations without .describe()', + run: async ({ srcFiles, read }) => { + let { missing, scanned } = await countMissingZodDescribes(srcFiles, read); + if (missing === 0) return { failed: false, detail: '' }; + let pct = scanned > 0 ? Math.round((missing / scanned) * 100) : 0; + return { + failed: true, + detail: `${missing}/${scanned} field${scanned === 1 ? '' : 's'} (${pct}%) missing .describe()` + }; + } + }, + { + name: 'readme', + severity: 'info', + description: 'integration is missing a README.md', + run: async ({ hasFile }) => { + if (await hasFile('README.md')) return { failed: false, detail: '' }; + return { failed: true, detail: 'no README.md' }; + } + } +]; + +export let buildCheckContext = async ( + integration: WorkspaceIntegrationSummary +): Promise => { + let srcDir = path.join(integration.dirPath, 'src'); + let srcFiles = await walkTsFiles(srcDir); + let resolve = (relative: string) => + path.isAbsolute(relative) ? relative : path.join(integration.dirPath, relative); + return { + integration, + srcFiles, + hasFile: relative => pathExists(resolve(relative)), + read: relative => readFile(resolve(relative), 'utf-8'), + readJson: async relative => JSON.parse(await readFile(resolve(relative), 'utf-8')) + }; +}; diff --git a/packages/cli/src/commands/doctor.test.ts b/packages/cli/src/commands/doctor.test.ts new file mode 100644 index 0000000000..edfa8e73c4 --- /dev/null +++ b/packages/cli/src/commands/doctor.test.ts @@ -0,0 +1,341 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import path from 'path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { runDoctor } from './doctor'; + +let tempDirs: string[] = []; + +let createTempDir = async () => { + let dir = await mkdtemp(path.join(tmpdir(), 'slates-cli-doctor-')); + tempDirs.push(dir); + return dir; +}; + +let writeIntegration = async ( + cwd: string, + name: string, + files: Record, + opts: { root?: string; packageJson?: Record } = {} +) => { + let root = opts.root ?? 'integrations'; + let dir = path.join(cwd, root, name); + await mkdir(path.join(dir, 'src'), { recursive: true }); + let manifest = { + name: `@slates/${name}`, + description: `${name} integration`, + author: 'Test Author', + license: 'FSL 1.1', + source: 'src/index.ts', + main: 'src/index.ts', + ...(opts.packageJson ?? {}) + }; + await writeFile( + path.join(dir, 'package.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + 'utf-8' + ); + for (let [relative, content] of Object.entries(files)) { + let target = path.join(dir, relative); + await mkdir(path.dirname(target), { recursive: true }); + await writeFile(target, content, 'utf-8'); + } +}; + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); +}); + +let getCheck = (report: NonNullable>>, name: string) => { + let match = report.checks.find(check => check.name === name); + if (!match) throw new Error(`Check ${name} missing from report`); + return match; +}; + +describe('runDoctor - raw-throws', () => { + it('flags integrations with raw throws and no lib/errors.ts', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'demo', { + 'src/index.ts': 'export let provider = {};\n', + 'src/tools.ts': "throw new Error('oops');\n" + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'raw-throws').failures).toEqual([ + { integration: 'demo', detail: '1 raw throw (no lib/errors.ts)' } + ]); + }); + + it('does not flag integrations using lib/errors.ts', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'clean', { + 'src/index.ts': 'export let provider = {};\n', + 'src/lib/errors.ts': 'export let cleanError = (msg: string) => msg;\n', + 'src/tools.ts': "import { cleanError } from './lib/errors';\nlet x = cleanError('hi');\n" + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'raw-throws').failures).toEqual([]); + }); + + it('flags partial migrations when raw throws remain alongside lib/errors.ts', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'partial', { + 'src/index.ts': 'export let provider = {};\n', + 'src/lib/errors.ts': 'export let partialError = (msg: string) => msg;\n', + 'src/tools.ts': "throw new Error('still raw');\nthrow new Error('twice');\n" + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'raw-throws').failures).toEqual([ + { integration: 'partial', detail: '2 raw throws (partial: lib/errors.ts exists)' } + ]); + }); + + it('ignores raw throws inside *.test.ts files', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'testy', { + 'src/index.ts': 'export let provider = {};\n', + 'src/index.test.ts': "throw new Error('test scaffold throw');\n" + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'raw-throws').failures).toEqual([]); + }); +}); + +describe('runDoctor - scope-file', () => { + it('flags OAuth integrations missing src/scopes.ts', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'oauth-only', { + 'src/index.ts': 'export let provider = {};\n', + 'src/auth.ts': "// Google OAuth flow\nexport let auth = { method: 'oauth' };\n" + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'scope-file').failures).toEqual([ + { integration: 'oauth-only', detail: 'OAuth detected in auth.ts but no scopes.ts' } + ]); + }); + + it('does not flag non-OAuth integrations even without scopes.ts', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'api-key', { + 'src/index.ts': 'export let provider = {};\n', + 'src/auth.ts': 'export let auth = { method: "api_key" };\n' + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'scope-file').failures).toEqual([]); + }); + + it('does not flag OAuth integrations that have scopes.ts', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'good-oauth', { + 'src/index.ts': 'export let provider = {};\n', + 'src/auth.ts': '// OAuth-based\nexport let auth = {};\n', + 'src/scopes.ts': 'export let scopes = {};\n' + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'scope-file').failures).toEqual([]); + }); +}); + +describe('runDoctor - contract-tests', () => { + it('flags integrations with auth.ts but no contract test', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'untested', { + 'src/index.ts': 'export let provider = {};\n', + 'src/auth.ts': 'export let auth = {};\n' + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'contract-tests').failures).toEqual([ + { integration: 'untested', detail: 'auth.ts present, no contract tests' } + ]); + }); + + it('does not flag integrations with contract tests', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'tested', { + 'src/index.ts': 'export let provider = {};\n', + 'src/auth.ts': 'export let auth = {};\n', + 'src/auth.contract.test.ts': "import { it } from 'vitest';\nit('passes', () => {});\n", + 'vitest.config.ts': 'export default { test: {} };\n' + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'contract-tests').failures).toEqual([]); + }); +}); + +describe('runDoctor - vitest-config', () => { + it('flags test files without a vitest.config.ts', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'no-config', { + 'src/index.ts': 'export let provider = {};\n', + 'src/index.test.ts': "import { it } from 'vitest';\nit('runs', () => {});\n" + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'vitest-config').failures).toEqual([ + { integration: 'no-config', detail: 'tests present, no vitest.config.ts' } + ]); + }); +}); + +describe('runDoctor - zod-describe', () => { + it('flags Zod field declarations missing .describe()', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'undescribed', { + 'src/index.ts': + "import { z } from 'zod';\nexport let schema = z.object({\n one: z.string(),\n two: z.number()\n});\n" + }); + + let report = await runDoctor({ cwd, json: true }); + let zod = getCheck(report!, 'zod-describe').failures; + expect(zod).toHaveLength(1); + expect(zod[0]!.integration).toBe('undescribed'); + expect(zod[0]!.detail).toMatch(/2\/2 fields \(100%\) missing \.describe\(\)/); + }); + + it('does not flag schemas where every field has .describe()', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'described', { + 'src/index.ts': + "import { z } from 'zod';\nexport let schema = z.object({\n one: z.string().describe('first'),\n two: z.number().optional().describe('second')\n});\n" + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'zod-describe').failures).toEqual([]); + }); + + it('handles multi-line chained schemas via line collapse', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'chained', { + 'src/index.ts': + "import { z } from 'zod';\nexport let schema = z.object({\n one: z\n .string()\n .optional()\n .describe('first')\n});\n" + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'zod-describe').failures).toEqual([]); + }); +}); + +describe('runDoctor - readme', () => { + it('flags integrations without a README.md', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'no-readme', { + 'src/index.ts': 'export let provider = {};\n' + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'readme').failures).toEqual([ + { integration: 'no-readme', detail: 'no README.md' } + ]); + }); + + it('does not flag integrations that have a README.md', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'documented', { + 'src/index.ts': 'export let provider = {};\n', + 'README.md': '# documented\n' + }); + + let report = await runDoctor({ cwd, json: true }); + expect(getCheck(report!, 'readme').failures).toEqual([]); + }); +}); + +describe('runDoctor - workspace selection', () => { + it('skips test-integrations by default', async () => { + let cwd = await createTempDir(); + await writeIntegration( + cwd, + 'fixture', + { 'src/tools.ts': "throw new Error('oops');\n" }, + { root: 'test-integrations' } + ); + + let report = await runDoctor({ cwd, json: true }); + expect(report!.auditedIntegrations).toBe(0); + }); + + it('includes test-integrations when includeTestIntegrations is true', async () => { + let cwd = await createTempDir(); + await writeIntegration( + cwd, + 'fixture', + { 'src/tools.ts': "throw new Error('oops');\n" }, + { root: 'test-integrations' } + ); + + let report = await runDoctor({ cwd, json: true, includeTestIntegrations: true }); + expect(report!.auditedIntegrations).toBe(1); + }); + + it('throws on unknown --check name', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'demo', { 'src/index.ts': 'export let provider = {};\n' }); + + await expect(runDoctor({ cwd, json: true, check: 'fake-check' })).rejects.toThrow( + /Unknown check/ + ); + }); + + it('throws when --integration matches no workspace integration', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'demo', { 'src/index.ts': 'export let provider = {};\n' }); + + await expect(runDoctor({ cwd, json: true, integration: 'missing' })).rejects.toThrow( + /No integration named "missing"/ + ); + }); + + it('filters by --integration name', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'one', { 'src/tools.ts': "throw new Error('a');\n" }); + await writeIntegration(cwd, 'two', { 'src/tools.ts': "throw new Error('b');\n" }); + + let report = await runDoctor({ cwd, json: true, integration: 'one' }); + expect(report!.auditedIntegrations).toBe(1); + expect(getCheck(report!, 'raw-throws').failures.map(f => f.integration)).toEqual(['one']); + }); + + it('returns only the requested check when --check is set', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'demo', { + 'src/auth.ts': 'export let auth = {};\n', + 'src/tools.ts': "throw new Error('oops');\n" + }); + + let report = await runDoctor({ cwd, json: true, check: 'contract-tests' }); + expect(report!.checks).toHaveLength(1); + expect(report!.checks[0]!.name).toBe('contract-tests'); + }); +}); + +describe('runDoctor - severity', () => { + it('attaches severity to every check in the JSON report', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'demo', { 'src/index.ts': 'export let provider = {};\n' }); + + let report = await runDoctor({ cwd, json: true }); + let severities = new Set(report!.checks.map(check => check.severity)); + expect(severities).toEqual(new Set(['error', 'warn', 'info'])); + }); + + it('tallies totals by severity in the JSON report', async () => { + let cwd = await createTempDir(); + await writeIntegration(cwd, 'demo', { + 'src/index.ts': 'export let provider = {};\n', + 'src/tools.ts': "throw new Error('oops');\n" + }); + + let report = await runDoctor({ cwd, json: true }); + expect(report!.totalsBySeverity.error).toBeGreaterThanOrEqual(1); + expect(report!.totalsBySeverity.warn).toBeGreaterThanOrEqual(0); + expect(report!.totalsBySeverity.info).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/packages/cli/src/commands/doctor.ts b/packages/cli/src/commands/doctor.ts new file mode 100644 index 0000000000..63d9d36dc2 --- /dev/null +++ b/packages/cli/src/commands/doctor.ts @@ -0,0 +1,209 @@ +import { + listWorkspaceIntegrations, + type WorkspaceIntegrationSummary +} from '../lib/integration'; +import { + buildCheckContext, + CHECKS, + type CheckSpec, + SEVERITY_ORDER, + type Severity +} from './doctor-checks'; + +export interface DoctorOptions { + check?: string; + integration?: string; + json?: boolean; + all?: boolean; + includeTestIntegrations?: boolean; + noColor?: boolean; + cwd?: string; +} + +export interface DoctorCheckReport { + name: string; + severity: Severity; + description: string; + failures: Array<{ integration: string; detail: string }>; +} + +export interface DoctorReport { + auditedIntegrations: number; + checks: DoctorCheckReport[]; + totalFailures: number; + totalsBySeverity: Record; +} + +interface Finding { + check: string; + severity: Severity; + integration: string; + detail: string; +} + +let PRINT_LIMIT = 10; +let SEVERITY_COLOR: Record = { + error: '31', + warn: '33', + info: '36' +}; +let SEVERITY_BADGE: Record = { + error: 'ERR', + warn: 'WRN', + info: 'INF' +}; + +let collectFindings = async ( + integrations: WorkspaceIntegrationSummary[], + activeChecks: CheckSpec[] +): Promise => { + let findings: Finding[] = []; + await Promise.all( + integrations.map(async integration => { + let ctx = await buildCheckContext(integration); + for (let check of activeChecks) { + let result = await check.run(ctx); + if (result?.failed) { + findings.push({ + check: check.name, + severity: check.severity, + integration: integration.name, + detail: result.detail + }); + } + } + }) + ); + findings.sort((left, right) => { + if (left.severity !== right.severity) { + return SEVERITY_ORDER[left.severity] - SEVERITY_ORDER[right.severity]; + } + if (left.check !== right.check) return left.check.localeCompare(right.check); + return left.integration.localeCompare(right.integration); + }); + return findings; +}; + +let tallyBySeverity = (findings: Finding[]) => + findings.reduce>( + (acc, finding) => { + acc[finding.severity]++; + return acc; + }, + { error: 0, warn: 0, info: 0 } + ); + +let useColor = (opts: DoctorOptions) => + !opts.noColor && !opts.json && Boolean(process.stdout.isTTY); + +let paint = (enabled: boolean, code: string, text: string) => + enabled ? `\x1b[${code}m${text}\x1b[0m` : text; + +let formatBadge = (severity: Severity, hasFailures: boolean, color: boolean) => { + if (!hasFailures) return paint(color, '32', ' ok'); + return paint(color, SEVERITY_COLOR[severity], SEVERITY_BADGE[severity]); +}; + +let renderPretty = ( + integrations: WorkspaceIntegrationSummary[], + activeChecks: CheckSpec[], + findings: Finding[], + opts: DoctorOptions +) => { + let color = useColor(opts); + let drillDown = Boolean(opts.check || opts.integration); + let lines: string[] = [ + '', + `Slates Doctor - ${integrations.length} integration${integrations.length === 1 ? '' : 's'} audited`, + '' + ]; + + let sortedChecks = [...activeChecks].sort( + (left, right) => SEVERITY_ORDER[left.severity] - SEVERITY_ORDER[right.severity] + ); + + for (let check of sortedChecks) { + let checkFindings = findings.filter(finding => finding.check === check.name); + let badge = formatBadge(check.severity, checkFindings.length > 0, color); + let count = checkFindings.length.toString().padStart(5); + lines.push(`${badge} ${check.name.padEnd(18)} ${count} ${check.description}`); + + if (drillDown && checkFindings.length > 0) { + let toShow = opts.all ? checkFindings : checkFindings.slice(0, PRINT_LIMIT); + for (let finding of toShow) { + lines.push(` - ${finding.integration.padEnd(32)} ${finding.detail}`); + } + let remaining = checkFindings.length - toShow.length; + if (remaining > 0) { + lines.push(` ... and ${remaining} more (pass --all to see all)`); + } + } + } + + let totals = tallyBySeverity(findings); + lines.push(''); + lines.push( + `Total: ${findings.length} finding${findings.length === 1 ? '' : 's'} (${totals.error} error, ${totals.warn} warn, ${totals.info} info)` + ); + + if (!drillDown && findings.length > 0) { + lines.push(''); + lines.push( + 'Pass --check= to drill into a check, --integration= to filter, --json for machine output.' + ); + } + + console.log(lines.join('\n')); +}; + +export let runDoctor = async (opts: DoctorOptions = {}): Promise => { + let integrations = await listWorkspaceIntegrations({ cwd: opts.cwd }); + if (integrations.length === 0) { + throw new Error('No integrations directory was found in the current repository.'); + } + + let activeChecks = opts.check ? CHECKS.filter(check => check.name === opts.check) : CHECKS; + if (opts.check && activeChecks.length === 0) { + let known = CHECKS.map(check => check.name).join(', '); + throw new Error(`Unknown check "${opts.check}". Known checks: ${known}.`); + } + + let filteredIntegrations = integrations.filter(integration => { + if (opts.integration && integration.name !== opts.integration) return false; + if ( + !opts.includeTestIntegrations && + integration.relativeDir.startsWith('test-integrations/') + ) { + return false; + } + return true; + }); + + if (opts.integration && filteredIntegrations.length === 0) { + throw new Error(`No integration named "${opts.integration}" was found.`); + } + + let findings = await collectFindings(filteredIntegrations, activeChecks); + + if (opts.json) { + return { + auditedIntegrations: filteredIntegrations.length, + checks: activeChecks.map(check => ({ + name: check.name, + severity: check.severity, + description: check.description, + failures: findings + .filter(finding => finding.check === check.name) + .map(finding => ({ + integration: finding.integration, + detail: finding.detail + })) + })), + totalFailures: findings.length, + totalsBySeverity: tallyBySeverity(findings) + }; + } + + renderPretty(filteredIntegrations, activeChecks, findings, opts); + return undefined; +}; diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 04d8728b77..4385d7502b 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -1,5 +1,6 @@ export * from './auth'; export * from './config'; +export * from './doctor'; export * from './profiles'; export * from './repl'; export * from './test'; diff --git a/packages/cli/src/lib/integration.ts b/packages/cli/src/lib/integration.ts index ca40f3b8e8..1c0d02a62d 100644 --- a/packages/cli/src/lib/integration.ts +++ b/packages/cli/src/lib/integration.ts @@ -20,7 +20,7 @@ export interface WorkspaceIntegrationSummary { let toPosixPath = (value: string) => value.replace(/\\/g, '/'); -let pathExists = async (targetPath: string) => { +export let pathExists = async (targetPath: string) => { try { await access(targetPath); return true;