Handle model provider timeouts and errors

.changeset/model-provider-errors.md

@@ -0,0 +1,6 @@
+---
+"@electric-ax/agents-runtime": patch
+"@electric-ax/agents-server-ui": patch
+---
+
+Add default model-provider timeout/error handling for agent runs and render durable run errors in the UI.

packages/agents-runtime/src/context-factory.ts

@@ -546,6 +546,9 @@ export function createHandlerContext<TState extends StateProxy = StateProxy>(
           getApiKey: activeAgentConfig.getApiKey,
 
           onPayload: activeAgentConfig.onPayload,
+
+          modelTimeoutMs: activeAgentConfig.modelTimeoutMs,
+          modelMaxRetries: activeAgentConfig.modelMaxRetries,
         })
         const handle = adapterFactory({
           entityUrl: config.entityUrl,

packages/agents-runtime/src/index.ts

@@ -163,6 +163,13 @@ export type {
 export { buildSections, buildTimelineEntries } from './use-chat'
 export type { EntityTimelineEntry } from './use-chat'
 export { appendPathToUrl } from './url'
+export {
+  ModelProviderError,
+  classifyModelProviderError,
+  modelProviderErrorMessage,
+  toModelProviderError,
+} from './model-provider-error'
+export type { ModelProviderErrorCode } from './model-provider-error'
 
 export {
   defaultProjection,

packages/agents-runtime/src/model-provider-error.ts

@@ -0,0 +1,145 @@
+export type ModelProviderErrorCode =
+  | `MODEL_PROVIDER_TIMEOUT`
+  | `MODEL_PROVIDER_UNREACHABLE`
+  | `MODEL_PROVIDER_AUTH_FAILED`
+  | `MODEL_PROVIDER_RATE_LIMITED`
+  | `MODEL_PROVIDER_UNAVAILABLE`
+  | `MODEL_PROVIDER_ERROR`
+
+export class ModelProviderError extends Error {
+  readonly code: ModelProviderErrorCode
+  readonly provider?: string
+  readonly model?: string
+
+  constructor(opts: {
+    code: ModelProviderErrorCode
+    message: string
+    provider?: string
+    model?: string
+    cause?: unknown
+  }) {
+    super(
+      opts.message,
+      opts.cause === undefined ? undefined : { cause: opts.cause }
+    )
+    this.name = `ModelProviderError`
+    this.code = opts.code
+    this.provider = opts.provider
+    this.model = opts.model
+  }
+}
+
+function stringifyError(error: unknown): string {
+  if (error instanceof Error) {
+    const cause = (error as { cause?: unknown }).cause
+    return [
+      error.name,
+      error.message,
+      cause === undefined ? `` : stringifyError(cause),
+    ]
+      .filter(Boolean)
+      .join(` `)
+  }
+  return String(error)
+}
+
+export function classifyModelProviderError(
+  error: unknown
+): ModelProviderErrorCode {
+  const text = stringifyError(error).toLowerCase()
+
+  if (
+    /\b(aborterror|timeouterror)\b/.test(text) ||
+    text.includes(`timeout`) ||
+    text.includes(`timed out`)
+  ) {
+    return `MODEL_PROVIDER_TIMEOUT`
+  }
+
+  if (
+    text.includes(`401`) ||
+    text.includes(`invalid api key`) ||
+    text.includes(`authentication`) ||
+    text.includes(`unauthorized`)
+  ) {
+    return `MODEL_PROVIDER_AUTH_FAILED`
+  }
+
+  if (text.includes(`429`) || text.includes(`rate limit`)) {
+    return `MODEL_PROVIDER_RATE_LIMITED`
+  }
+
+  if (
+    text.includes(`502`) ||
+    text.includes(`503`) ||
+    text.includes(`504`) ||
+    text.includes(`overloaded`) ||
+    text.includes(`unavailable`)
+  ) {
+    return `MODEL_PROVIDER_UNAVAILABLE`
+  }
+
+  if (
+    text.includes(`enotfound`) ||
+    text.includes(`econnrefused`) ||
+    text.includes(`econnreset`) ||
+    text.includes(`eai_again`) ||
+    text.includes(`fetch failed`) ||
+    text.includes(`failed to fetch`) ||
+    text.includes(`network`)
+  ) {
+    return `MODEL_PROVIDER_UNREACHABLE`
+  }
+
+  return `MODEL_PROVIDER_ERROR`
+}
+
+export function modelProviderErrorMessage(opts: {
+  code: ModelProviderErrorCode
+  provider?: string
+}): string {
+  const provider = opts.provider
+    ? displayProvider(opts.provider)
+    : `the model provider`
+  switch (opts.code) {
+    case `MODEL_PROVIDER_TIMEOUT`:
+      return `${provider} did not respond before the timeout. Check your Internet connection or provider status.`
+    case `MODEL_PROVIDER_UNREACHABLE`:
+      return `Could not reach ${provider}. Check your Internet connection or ${provider} status.`
+    case `MODEL_PROVIDER_AUTH_FAILED`:
+      return `${provider} rejected the API key. Check your model provider credentials.`
+    case `MODEL_PROVIDER_RATE_LIMITED`:
+      return `${provider} rate limited the request. Please wait and try again.`
+    case `MODEL_PROVIDER_UNAVAILABLE`:
+      return `${provider} is currently unavailable. Check provider status and try again.`
+    case `MODEL_PROVIDER_ERROR`:
+      return `${provider} returned an error. Check the runtime logs for provider details.`
+  }
+}
+
+export function toModelProviderError(
+  error: unknown,
+  opts: { provider?: string; model?: string }
+): ModelProviderError {
+  if (error instanceof ModelProviderError) return error
+  const code = classifyModelProviderError(error)
+  const detail = error instanceof Error ? error.message : String(error)
+  return new ModelProviderError({
+    code,
+    provider: opts.provider,
+    model: opts.model,
+    message: `${modelProviderErrorMessage({ code, provider: opts.provider })} (${detail})`,
+    cause: error,
+  })
+}
+
+function displayProvider(provider: string): string {
+  switch (provider.toLowerCase()) {
+    case `anthropic`:
+      return `Anthropic`
+    case `openai`:
+      return `OpenAI`
+    default:
+      return provider
+  }
+}

packages/agents-runtime/src/outbound-bridge.ts

@@ -100,6 +100,7 @@ export async function loadOutboundIdSeed(
 export interface OutboundBridge {
   onRunStart: () => void
   onRunEnd: (opts?: { finishReason?: string }) => void
+  onError: (opts: { errorCode: string; message: string }) => void
   onStepStart: (opts?: { modelProvider?: string; modelId?: string }) => void
   onStepEnd: (opts?: {
     finishReason?: string
@@ -193,6 +194,21 @@ export function createOutboundBridge(
       currentRunKey = null
     },
 
+    onError(opts: { errorCode: string; message: string }) {
+      if (!currentRunKey) return
+      writeEvent(
+        entityStateSchema.errors.insert({
+          key: `${currentRunKey}:error-${crypto.randomUUID()}`,
+          value: {
+            error_code: opts.errorCode,
+            message: opts.message,
+            run_id: currentRunKey,
+            ...(currentStepKey ? { step_id: currentStepKey } : {}),
+          } as never,
+        }) as ChangeEvent
+      )
+    },
+
     onStepStart(opts?: { modelProvider?: string; modelId?: string }) {
       const runKey = requireActiveRun(`onStepStart`)
       currentStepKey = `step-${counters.step++}`

packages/agents-runtime/src/pi-adapter.ts

@@ -8,10 +8,14 @@
  */
 
 import { Agent } from '@mariozechner/pi-agent-core'
-import { getModel } from '@mariozechner/pi-ai'
+import { getModel, streamSimple } from '@mariozechner/pi-ai'
 import { createOutboundBridge } from './outbound-bridge'
 import { MOONSHOT_PROVIDER, getMoonshotModel } from './moonshot-models'
 import { runtimeLog } from './log'
+import {
+  ModelProviderError,
+  toModelProviderError,
+} from './model-provider-error'
 import type { OutboundIdSeed } from './outbound-bridge'
 import type { ChangeEvent } from '@durable-streams/state'
 import type {
@@ -42,8 +46,13 @@ export interface PiAdapterOptions {
     provider: string
   ) => Promise<string | undefined> | string | undefined
   onPayload?: SimpleStreamOptions[`onPayload`]
+  modelTimeoutMs?: number
+  modelMaxRetries?: number
 }
 
+const DEFAULT_MODEL_TIMEOUT_MS = 30_000
+const DEFAULT_MODEL_MAX_RETRIES = 0
+
 interface PiAgentAdapterConfig {
   entityUrl: string
   epoch: number
@@ -227,18 +236,32 @@ export function createPiAgentAdapter(
       model: opts.model,
       ...(opts.provider && { provider: opts.provider }),
     })
+    const modelTimeoutMs = opts.modelTimeoutMs ?? DEFAULT_MODEL_TIMEOUT_MS
+    const modelMaxRetries = opts.modelMaxRetries ?? DEFAULT_MODEL_MAX_RETRIES
+
+    const baseStreamFn = opts.streamFn ?? streamSimple
+    const streamFn: StreamFn = (streamModel, context, streamOptions) =>
+      baseStreamFn(streamModel, context, {
+        ...streamOptions,
+        timeoutMs: modelTimeoutMs,
+        maxRetries: modelMaxRetries,
+      })
 
-    const agent = new Agent({
+    const agentOptions = {
       initialState: {
         systemPrompt: opts.systemPrompt,
         tools: opts.tools as Array<never>,
         messages: history as Array<never>,
         model,
       },
-      ...(opts.streamFn && { streamFn: opts.streamFn }),
+      streamFn,
       ...(opts.getApiKey && { getApiKey: opts.getApiKey }),
       ...(opts.onPayload && { onPayload: opts.onPayload }),
-    })
+    }
+
+    const agent = new Agent(
+      agentOptions as ConstructorParameters<typeof Agent>[0]
+    )
 
     function processAgentEvents(
       resolveWhenDone: () => void,
@@ -361,8 +384,11 @@ export function createPiAgentAdapter(
                 })
 
                 if (isError) {
-                  throw new Error(
-                    `pi-agent message_end error: ${msg.errorMessage ?? `unknown error`} (stopReason=${msg.stopReason ?? `none`})`
+                  throw toModelProviderError(
+                    new Error(
+                      `pi-agent message_end error: ${msg.errorMessage ?? `unknown error`} (stopReason=${msg.stopReason ?? `none`})`
+                    ),
+                    { provider: model.provider, model: model.id }
                   )
                 }
                 break
@@ -388,6 +414,20 @@ export function createPiAgentAdapter(
               }
 
               case `agent_end`: {
+                const messages = (event as Record<string, unknown>).messages as
+                  | Array<{ stopReason?: string; errorMessage?: string }>
+                  | undefined
+                const errorMessage = messages?.find(
+                  (message) =>
+                    message.stopReason === `error` && !!message.errorMessage
+                )?.errorMessage
+                if (errorMessage) {
+                  throw toModelProviderError(
+                    new Error(`pi-agent agent_end error: ${errorMessage}`),
+                    { provider: model.provider, model: model.id }
+                  )
+                }
+
                 bridge.onRunEnd({
                   finishReason: abortedRun ? `aborted` : `stop`,
                 })
@@ -451,6 +491,17 @@ export function createPiAgentAdapter(
             unsubscribe()
             bridge.onRunEnd({ finishReason })
           }
+          const failWithProviderError = (err: unknown): ModelProviderError => {
+            const providerError = toModelProviderError(err, {
+              provider: model.provider,
+              model: model.id,
+            })
+            bridge.onError({
+              errorCode: providerError.code,
+              message: providerError.message,
+            })
+            return providerError
+          }
           const abortRun = (): void => {
             if (settled) return
             abortedRun = true
@@ -476,8 +527,9 @@ export function createPiAgentAdapter(
             },
             (err) => {
               if (settled) return
+              const providerError = failWithProviderError(err)
               finish(`error`)
-              reject(err)
+              reject(providerError)
             }
           )
 
@@ -491,8 +543,9 @@ export function createPiAgentAdapter(
           Promise.resolve(runPromise).catch((err: Error) => {
             if (settled) return
             if (abortedRun) return
+            const providerError = failWithProviderError(err)
             finish(`error`)
-            reject(err)
+            reject(providerError)
           })
         })
       },

packages/agents-runtime/src/process-wake.ts

@@ -13,6 +13,7 @@ import { unrestrictedSandbox } from './sandbox/unrestricted'
 import { resolveSandboxIdentity } from './sandbox/identity'
 import { appendPathToUrl } from './url'
 import { manifestChildKey } from './manifest-helpers'
+import { ModelProviderError } from './model-provider-error'
 import {
   buildHydratedEventSourceWake,
   eventSourceWakeInfoFromManifests,
@@ -2078,13 +2079,18 @@ export async function processWake(
         await waitForSignalHandlers()
         activeSignalHandler = null
         wakeSession.rollbackManifestEntries()
-        const errMsg = toError(setupErr).message
+        const err = toError(setupErr)
+        const errMsg = err.message
+        const errCode =
+          setupErr instanceof ModelProviderError
+            ? setupErr.code
+            : `HANDLER_FAILED`
         log.error(`handler failed for ${entityUrl}:`, errMsg)
         writeEvent(
           entityStateSchema.errors.insert({
             key: `error-${epoch}-${crypto.randomUUID()}`,
             value: {
-              error_code: `HANDLER_FAILED`,
+              error_code: errCode,
               message: errMsg,
             } as never,
           }) as ChangeEvent

packages/agents-runtime/src/types.ts

@@ -858,6 +858,8 @@ export interface AgentConfig {
     provider: string
   ) => Promise<string | undefined> | string | undefined
   onPayload?: SimpleStreamOptions[`onPayload`]
+  modelTimeoutMs?: number
+  modelMaxRetries?: number
   testResponses?: TestResponses
 }