Add slates doctor command

packages/cli/README.md

@@ -0,0 +1,135 @@
+# @slates/cli
+
+Reference CLI for the Metorial integrations workspace. Provides per-integration
+and global commands for setup, auth, tool invocation, profile management, and
+workspace-wide audits.
+
+## Install
+
+The CLI runs through Bun directly from the workspace:
+
+```bash
+bun packages/cli/src/cli.ts <command>
+```
+
+Or via the root-level alias:
+
+```bash
+bun run integrations:cli <command>
+```
+
+## Global commands
+
+### `slates test`
+
+Run vitest across every integration in the workspace.
+
+```bash
+bun run integrations:cli test
+bun run integrations:cli test -- --reporter=verbose
+```
+
+Arguments after `--` are forwarded to vitest.
+
+### `slates doctor`
+
+Workspace-wide consistency audit. Walks every `integrations/<name>` folder and
+reports gaps against the standards used in the canonical Google-family
+integrations (`gmail`, `google-meet`, `youtube`, `google-analytics`).
+
+```bash
+bun run integrations:cli doctor
+bun run integrations:cli doctor --check=raw-throws --all
+bun run integrations:cli doctor --integration=linear
+bun run integrations:cli doctor --json > doctor.json
+```
+
+#### Checks
+
+| Check | Severity | What it catches |
+|-------|----------|------------------|
+| `raw-throws` | `error` | `throw new Error(...)` outside test files. Tags partial migrations where `src/lib/errors.ts` already exists. |
+| `scope-file` | `error` | Integration has `src/auth.ts` containing OAuth logic but no `src/scopes.ts`. |
+| `vitest-config` | `error` | Test files present but no `vitest.config.ts`. |
+| `contract-tests` | `warn` | `src/auth.ts` present but no `*.contract.test.ts`. |
+| `zod-describe` | `warn` | Top-level Zod field declarations without `.describe()`. Multi-line chains handled. |
+| `readme` | `info` | Integration has no `README.md`. |
+
+Severity bucketing:
+
+- `error` — breaks `@lowerdeck/error` tagging, runtime behavior, or test discovery.
+- `warn` — degrades AI tool-calling quality, test coverage, or platform integration.
+- `info` — best-practice gap, usually documentation or metadata.
+
+#### Flags
+
+| Flag | Effect |
+|------|--------|
+| `--check=<name>` | Run only the named check. Drills into the per-integration breakdown. |
+| `--integration=<name>` | Limit the audit to a single integration directory. |
+| `--all` | Show every failing integration (default: top 10 per check). |
+| `--json` | Emit a machine-readable report instead of the pretty table. |
+| `--no-color` | Disable ANSI color in pretty output (default: auto-detected from TTY). |
+| `--include-test-integrations` | Include `test-integrations/` in the audit. |
+
+#### Example output
+
+```
+Slates Doctor - 1121 integrations audited
+
+ERR  raw-throws           584  src/ has raw `throw new Error(...)` - use lib/errors.ts helpers
+ERR  scope-file           251  OAuth integration without src/scopes.ts
+ ok  vitest-config          0  test files present but no vitest.config.ts
+WRN  contract-tests      1092  src/auth.ts present but no *.contract.test.ts
+WRN  zod-describe        1119  Zod field declarations without .describe()
+ ok  readme                 0  integration is missing a README.md
+
+Total: 3046 findings (835 error, 2211 warn, 0 info)
+```
+
+#### JSON shape
+
+```jsonc
+{
+  "auditedIntegrations": 1121,
+  "totalFailures": 3176,
+  "totalsBySeverity": { "error": 630, "warn": 1296, "info": 1230 },
+  "checks": [
+    {
+      "name": "raw-throws",
+      "severity": "error",
+      "description": "src/ has raw `throw new Error(...)` - use lib/errors.ts helpers",
+      "failures": [
+        { "integration": "optimizely", "detail": "69 raw throws (no lib/errors.ts)" }
+      ]
+    }
+  ]
+}
+```
+
+#### Relationship to `scripts/validate-pr-integrations.ts`
+
+`validate-pr-integrations.ts` validates **schema diffs for paths changed in a
+PR** by building both base and head, capturing snapshots, and comparing
+provider / tool / auth-method schemas. It is the per-PR safety gate.
+
+`slates doctor` complements it by auditing **code-shape consistency across the
+whole workspace** without any build step. Use both: doctor for periodic health
+snapshots and standards tracking, `validate-pr-integrations` for per-PR
+behavioral safety.
+
+## Per-integration commands
+
+Invoked as `slates <integration> <command>`, where `<integration>` is the name
+of a directory under `integrations/` or `test-integrations/`.
+
+- `slates <integration> setup` — interactive integration setup
+- `slates <integration> profiles {add,list,get,use,remove}` — profile management
+- `slates <integration> tools {list,get,schema,call}` — invoke tools
+- `slates <integration> auth {list,get,setup,refresh,credentials}` — manage auth state
+- `slates <integration> config {get,set,schema}` — manage runtime config
+- `slates <integration> test` — run vitest against this integration with a profile context
+- `slates <integration> repl` — open an interactive REPL with the loaded provider
+
+Run `bun run integrations:cli <integration> --help` for the full per-command
+flag list.

packages/cli/src/cli.ts

@@ -17,6 +17,7 @@ import {
   refreshAuth,
   removeProfile,
   runAllIntegrationTests,
+  runDoctor,
   runVitestWithProfile,
   setConfig,
   setupAuth,
@@ -39,15 +40,18 @@ let printResult = async (cb: () => Promise<unknown>) => {
 
 let cli = sade('slates');
 let argv = process.argv.slice(2);
-let isGlobalTestCommand = argv[0] === 'test';
-let integration = isGlobalTestCommand ? null : argv[0];
+let GLOBAL_COMMANDS = new Set(['test', 'doctor']);
+let isGlobalCommand = GLOBAL_COMMANDS.has(argv[0] ?? '');
+let integration = isGlobalCommand ? null : argv[0];
 
-if (!isGlobalTestCommand && (!integration || integration.startsWith('-'))) {
-  console.error('Usage: slates <integration> <command>\n       slates test');
+if (!isGlobalCommand && (!integration || integration.startsWith('-'))) {
+  console.error(
+    'Usage: slates <integration> <command>\n       slates test\n       slates doctor'
+  );
   process.exit(1);
 }
 
-if (isGlobalTestCommand) {
+if (isGlobalCommand) {
   cli.command('test').action(() =>
     printResult(async () => {
       let separatorIndex = process.argv.indexOf('--');
@@ -57,6 +61,31 @@ if (isGlobalTestCommand) {
     })
   );
 
+  cli
+    .command('doctor')
+    .describe('Audit every integration in the workspace for consistency gaps.')
+    .option('--check', 'Run a single named check (see README for the full list)')
+    .option('--integration', 'Limit the audit to one integration by directory name')
+    .option('--json', 'Emit a machine-readable JSON report instead of the pretty table')
+    .option('--all', 'Show every failing integration instead of only the top 10')
+    .option('--no-color', 'Disable ANSI color in pretty output (auto-detected from TTY)')
+    .option(
+      '--include-test-integrations',
+      'Include integrations under test-integrations/ in the audit'
+    )
+    .action(opts =>
+      printResult(() =>
+        runDoctor({
+          check: opts.check,
+          integration: opts.integration,
+          json: Boolean(opts.json),
+          all: Boolean(opts.all),
+          noColor: opts.color === false,
+          includeTestIntegrations: Boolean(opts['include-test-integrations'])
+        })
+      )
+    );
+
   cli.parse([process.argv[0] ?? 'bun', process.argv[1] ?? 'slates', ...argv]);
 } else {
   cli

packages/cli/src/commands/doctor-checks.ts

@@ -0,0 +1,187 @@
+import { readdir, readFile } from 'fs/promises';
+import path from 'path';
+import { pathExists, type WorkspaceIntegrationSummary } from '../lib/integration';
+
+export type Severity = 'error' | 'warn' | 'info';
+
+export interface CheckContext {
+  integration: WorkspaceIntegrationSummary;
+  srcFiles: string[];
+  hasFile: (relative: string) => Promise<boolean>;
+  read: (relative: string) => Promise<string>;
+  readJson: <T = Record<string, unknown>>(relative: string) => Promise<T>;
+}
+
+export interface CheckResult {
+  failed: boolean;
+  detail: string;
+}
+
+export interface CheckSpec {
+  name: string;
+  severity: Severity;
+  description: string;
+  run: (ctx: CheckContext) => Promise<CheckResult | null>;
+}
+
+export let SEVERITY_ORDER: Record<Severity, number> = {
+  error: 0,
+  warn: 1,
+  info: 2
+};
+
+let SKIP_DIRS = new Set(['node_modules', 'dist', '.turbo']);
+let ZOD_FIELD_REGEX = /^\s+\w+:\s*z\.\w+\(/;
+let RAW_THROW_REGEX = /throw new Error\(/g;
+let OAUTH_PROBE_REGEX = /oauth/i;
+
+let walkTsFiles = async (dir: string): Promise<string[]> => {
+  if (!(await pathExists(dir))) return [];
+  let collected: string[] = [];
+  let stack = [dir];
+  while (stack.length > 0) {
+    let current = stack.pop()!;
+    let entries = await readdir(current, { withFileTypes: true });
+    for (let entry of entries) {
+      if (entry.isDirectory()) {
+        if (SKIP_DIRS.has(entry.name)) continue;
+        stack.push(path.join(current, entry.name));
+      } else if (
+        entry.isFile() &&
+        (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx'))
+      ) {
+        collected.push(path.join(current, entry.name));
+      }
+    }
+  }
+  return collected;
+};
+
+let collapseChains = (source: string) => source.replace(/\n\s*\./g, '.');
+
+let countRawThrows = async (srcFiles: string[], read: (file: string) => Promise<string>) => {
+  let count = 0;
+  await Promise.all(
+    srcFiles.map(async file => {
+      if (file.endsWith('.test.ts')) return;
+      let content = await read(file);
+      let matches = content.match(RAW_THROW_REGEX);
+      if (matches) count += matches.length;
+    })
+  );
+  return count;
+};
+
+let countMissingZodDescribes = async (
+  srcFiles: string[],
+  read: (file: string) => Promise<string>
+) => {
+  let missing = 0;
+  let scanned = 0;
+  await Promise.all(
+    srcFiles.map(async file => {
+      if (file.endsWith('.test.ts')) return;
+      let content = await read(file);
+      let collapsed = collapseChains(content);
+      for (let line of collapsed.split('\n')) {
+        if (!ZOD_FIELD_REGEX.test(line)) continue;
+        scanned++;
+        if (!line.includes('.describe(')) missing++;
+      }
+    })
+  );
+  return { missing, scanned };
+};
+
+export let CHECKS: CheckSpec[] = [
+  {
+    name: 'raw-throws',
+    severity: 'error',
+    description: 'src/ has raw `throw new Error(...)` - use lib/errors.ts helpers',
+    run: async ({ srcFiles, hasFile, read }) => {
+      let throwCount = await countRawThrows(srcFiles, read);
+      if (throwCount === 0) return { failed: false, detail: '' };
+      let hasErrorsHelper = await hasFile('src/lib/errors.ts');
+      let suffix = hasErrorsHelper
+        ? ' (partial: lib/errors.ts exists)'
+        : ' (no lib/errors.ts)';
+      return {
+        failed: true,
+        detail: `${throwCount} raw throw${throwCount === 1 ? '' : 's'}${suffix}`
+      };
+    }
+  },
+  {
+    name: 'scope-file',
+    severity: 'error',
+    description: 'OAuth integration without src/scopes.ts',
+    run: async ({ hasFile, read }) => {
+      if (!(await hasFile('src/auth.ts'))) return null;
+      let authContent = await read('src/auth.ts');
+      if (!OAUTH_PROBE_REGEX.test(authContent)) return null;
+      if (await hasFile('src/scopes.ts')) return { failed: false, detail: '' };
+      return { failed: true, detail: 'OAuth detected in auth.ts but no scopes.ts' };
+    }
+  },
+  {
+    name: 'vitest-config',
+    severity: 'error',
+    description: 'test files present but no vitest.config.ts',
+    run: async ({ srcFiles, hasFile }) => {
+      let hasTests = srcFiles.some(file => file.endsWith('.test.ts'));
+      if (!hasTests) return null;
+      if (await hasFile('vitest.config.ts')) return { failed: false, detail: '' };
+      return { failed: true, detail: 'tests present, no vitest.config.ts' };
+    }
+  },
+  {
+    name: 'contract-tests',
+    severity: 'warn',
+    description: 'src/auth.ts present but no *.contract.test.ts',
+    run: async ({ srcFiles, hasFile }) => {
+      if (!(await hasFile('src/auth.ts'))) return null;
+      let hasContractTest = srcFiles.some(file => file.endsWith('.contract.test.ts'));
+      if (hasContractTest) return { failed: false, detail: '' };
+      return { failed: true, detail: 'auth.ts present, no contract tests' };
+    }
+  },
+  {
+    name: 'zod-describe',
+    severity: 'warn',
+    description: 'Zod field declarations without .describe()',
+    run: async ({ srcFiles, read }) => {
+      let { missing, scanned } = await countMissingZodDescribes(srcFiles, read);
+      if (missing === 0) return { failed: false, detail: '' };
+      let pct = scanned > 0 ? Math.round((missing / scanned) * 100) : 0;
+      return {
+        failed: true,
+        detail: `${missing}/${scanned} field${scanned === 1 ? '' : 's'} (${pct}%) missing .describe()`
+      };
+    }
+  },
+  {
+    name: 'readme',
+    severity: 'info',
+    description: 'integration is missing a README.md',
+    run: async ({ hasFile }) => {
+      if (await hasFile('README.md')) return { failed: false, detail: '' };
+      return { failed: true, detail: 'no README.md' };
+    }
+  }
+];
+
+export let buildCheckContext = async (
+  integration: WorkspaceIntegrationSummary
+): Promise<CheckContext> => {
+  let srcDir = path.join(integration.dirPath, 'src');
+  let srcFiles = await walkTsFiles(srcDir);
+  let resolve = (relative: string) =>
+    path.isAbsolute(relative) ? relative : path.join(integration.dirPath, relative);
+  return {
+    integration,
+    srcFiles,
+    hasFile: relative => pathExists(resolve(relative)),
+    read: relative => readFile(resolve(relative), 'utf-8'),
+    readJson: async relative => JSON.parse(await readFile(resolve(relative), 'utf-8'))
+  };
+};