diff --git a/.plan/335-setup-validate.md b/.plan/335-setup-validate.md new file mode 100644 index 0000000..2a0c22d --- /dev/null +++ b/.plan/335-setup-validate.md @@ -0,0 +1,83 @@ +# Test Spec: uwf setup model connectivity validation (#335) + +## Context + +File: `packages/cli-workflow/src/commands/setup.ts` +Test file: `packages/cli-workflow/src/__tests__/setup-validate.test.ts` + +After `cmdSetup` writes config, it should send a test chat completion request to verify the configured model is reachable. If validation fails, warn the user (don't abort — config is already saved). + +## Implementation Notes + +- Add a `validateModel(baseUrl, apiKey, model)` function that sends a minimal chat completion request (`POST /chat/completions` with `messages: [{role:"user",content:"hi"}]`, `max_tokens: 1`) +- Returns `Result` — ok if 2xx response, error with reason string otherwise +- Use `AbortSignal.timeout(15_000)` for the request +- Both `cmdSetup` and `cmdSetupInteractive` should call it after saving config +- `cmdSetup` returns validation result in its return object: `{ ...existing, validation: { ok: true } | { ok: false, error: string } }` +- `cmdSetupInteractive` prints a warning to console if validation fails, success message if it passes +- Use the project logger (`createLogger`) — no raw `console.log` except in interactive CLI output (per CLAUDE.md) + +## Test Cases (vitest) + +### 1. `validateModel` — success path +- Mock `fetch` to return `{ status: 200, ok: true, json: () => ({}) }` +- Call `validateModel(baseUrl, apiKey, model)` +- Assert returns `{ ok: true, value: undefined }` +- Assert fetch was called with correct URL (`${baseUrl}/chat/completions`), correct headers (`Authorization: Bearer ${apiKey}`), correct body (model, messages, max_tokens: 1) + +### 2. `validateModel` — HTTP error (401 unauthorized) +- Mock `fetch` to return `{ status: 401, ok: false, statusText: "Unauthorized" }` +- Call `validateModel(baseUrl, apiKey, model)` +- Assert returns `{ ok: false, error: }` + +### 3. `validateModel` — HTTP error (404 model not found) +- Mock `fetch` to return `{ status: 404, ok: false, statusText: "Not Found" }` +- Assert returns `{ ok: false, error: }` + +### 4. `validateModel` — network timeout +- Mock `fetch` to throw `DOMException` with name `AbortError` +- Assert returns `{ ok: false, error: }` + +### 5. `validateModel` — network error (DNS failure, connection refused) +- Mock `fetch` to throw `TypeError("fetch failed")` +- Assert returns `{ ok: false, error: }` + +### 6. `cmdSetup` — includes validation result on success +- Mock global `fetch` for `/chat/completions` to succeed +- Call `cmdSetup({ provider, baseUrl, apiKey, model, storageRoot })` +- Assert returned object has `validation: { ok: true, value: undefined }` +- Assert config files are still written (existing behavior preserved) + +### 7. `cmdSetup` — includes validation result on failure (config still saved) +- Mock global `fetch` for `/chat/completions` to return 401 +- Call `cmdSetup({ ... })` +- Assert returned object has `validation: { ok: false, error: ... }` +- Assert `config.yaml` and `.env` are still written (validation failure doesn't prevent saving) + +### 8. `cmdSetupInteractive` — prints success message on validation pass +- Mock `fetch` for both `/models` and `/chat/completions` to succeed +- Mock stdin to provide valid selections +- Capture console output +- Assert output contains a success message like "Model verified" or "✓" + +### 9. `cmdSetupInteractive` — prints warning on validation failure +- Mock `fetch`: `/models` succeeds, `/chat/completions` returns 401 +- Mock stdin for valid selections +- Capture console output +- Assert output contains a warning about model not being reachable and suggests trying a different model + +### 10. `validateModel` — request body correctness +- Mock `fetch` to capture the request body +- Call `validateModel(baseUrl, apiKey, "test-model")` +- Assert body is `{ model: "test-model", messages: [{role: "user", content: "hi"}], max_tokens: 1 }` + +## Export Requirements + +- `validateModel` must be exported (for direct unit testing) +- Signature: `async function validateModel(baseUrl: string, apiKey: string, model: string): Promise>` +- `Result` type: `{ ok: true; value: T } | { ok: false; error: E }` (project convention) + +## Files to Create/Modify + +- **New**: `packages/cli-workflow/src/__tests__/setup-validate.test.ts` — all test cases above +- **Modify**: `packages/cli-workflow/src/commands/setup.ts` — add `validateModel`, integrate into `cmdSetup` and `cmdSetupInteractive` diff --git a/packages/cli-workflow/src/__tests__/setup-validate.test.ts b/packages/cli-workflow/src/__tests__/setup-validate.test.ts new file mode 100644 index 0000000..f96baa3 --- /dev/null +++ b/packages/cli-workflow/src/__tests__/setup-validate.test.ts @@ -0,0 +1,150 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { cmdSetup, validateModel } from "../commands/setup.js"; + +describe("validateModel", () => { + const BASE_URL = "https://api.example.com/v1"; + const API_KEY = "sk-test-key"; + const MODEL = "test-model"; + + afterEach(() => { + vi.restoreAllMocks(); + }); + + test("success path — returns ok on 200", async () => { + const mockFetch = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(JSON.stringify({}), { status: 200 })); + + const result = await validateModel(BASE_URL, API_KEY, MODEL); + + expect(result).toEqual({ ok: true, value: undefined }); + expect(mockFetch).toHaveBeenCalledOnce(); + + const [url, opts] = mockFetch.mock.calls[0]!; + expect(url).toBe(`${BASE_URL}/chat/completions`); + expect((opts as RequestInit).headers).toEqual( + expect.objectContaining({ Authorization: `Bearer ${API_KEY}` }), + ); + const body = JSON.parse((opts as RequestInit).body as string); + expect(body).toEqual({ + model: MODEL, + messages: [{ role: "user", content: "hi" }], + max_tokens: 1, + }); + }); + + test("HTTP 401 — returns error containing 401", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Unauthorized", { status: 401, statusText: "Unauthorized" }), + ); + + const result = await validateModel(BASE_URL, API_KEY, MODEL); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error).toContain("401"); + } + }); + + test("HTTP 404 — returns error containing 404", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Not Found", { status: 404, statusText: "Not Found" }), + ); + + const result = await validateModel(BASE_URL, API_KEY, MODEL); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error).toContain("404"); + } + }); + + test("network timeout — returns error mentioning timeout", async () => { + const err = new DOMException("signal timed out", "AbortError"); + vi.spyOn(globalThis, "fetch").mockRejectedValue(err); + + const result = await validateModel(BASE_URL, API_KEY, MODEL); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.toLowerCase()).toMatch(/timeout|timed out/); + } + }); + + test("network error (DNS/connection) — returns error mentioning connectivity", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new TypeError("fetch failed")); + + const result = await validateModel(BASE_URL, API_KEY, MODEL); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.toLowerCase()).toMatch(/connect|reach|network/); + } + }); + + test("request body correctness", async () => { + const mockFetch = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(JSON.stringify({}), { status: 200 })); + + await validateModel(BASE_URL, API_KEY, "my-special-model"); + + const body = JSON.parse((mockFetch.mock.calls[0]![1] as RequestInit).body as string); + expect(body).toEqual({ + model: "my-special-model", + messages: [{ role: "user", content: "hi" }], + max_tokens: 1, + }); + }); +}); + +describe("cmdSetup with validation", () => { + let storageRoot: string; + + beforeEach(async () => { + storageRoot = await mkdtemp(join(tmpdir(), "uwf-setup-validate-")); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + await rm(storageRoot, { recursive: true, force: true }); + }); + + const setupArgs = () => ({ + provider: "testprovider", + baseUrl: "https://api.test.com/v1", + apiKey: "sk-test", + model: "test-model", + storageRoot, + }); + + test("includes validation result on success", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({}), { status: 200 }), + ); + + const result = await cmdSetup(setupArgs()); + + expect(result.validation).toEqual({ ok: true, value: undefined }); + // Config files should still be written + expect(result.configPath).toBeTruthy(); + expect(result.envPath).toBeTruthy(); + }); + + test("includes validation failure — config still saved", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Unauthorized", { status: 401, statusText: "Unauthorized" }), + ); + + const result = await cmdSetup(setupArgs()); + + expect(result.validation).toBeDefined(); + expect((result.validation as { ok: boolean }).ok).toBe(false); + // Config files should still be written despite validation failure + expect(result.configPath).toBeTruthy(); + expect(result.envPath).toBeTruthy(); + }); +}); diff --git a/packages/cli-workflow/src/commands/setup.ts b/packages/cli-workflow/src/commands/setup.ts index 0eeab20..aa4ef61 100644 --- a/packages/cli-workflow/src/commands/setup.ts +++ b/packages/cli-workflow/src/commands/setup.ts @@ -2,9 +2,45 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { stdin as input, stdout as output } from "node:process"; import { createInterface } from "node:readline/promises"; - +import type { Result } from "@uncaged/workflow-util"; import { parse, stringify } from "yaml"; +/** + * Send a minimal chat completion request to verify the model is reachable. + * Returns ok on 2xx, error with reason string otherwise. + */ +export async function validateModel( + baseUrl: string, + apiKey: string, + model: string, +): Promise> { + try { + const url = `${baseUrl.replace(/\/+$/, "")}/chat/completions`; + const res = await fetch(url, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model, + messages: [{ role: "user", content: "hi" }], + max_tokens: 1, + }), + signal: AbortSignal.timeout(15_000), + }); + if (!res.ok) { + return { ok: false, error: `HTTP ${res.status} ${res.statusText}` }; + } + return { ok: true, value: undefined }; + } catch (err: unknown) { + if (err instanceof DOMException && err.name === "AbortError") { + return { ok: false, error: "Request timed out — model endpoint unreachable" }; + } + return { ok: false, error: `Network error — could not reach endpoint (${String(err)})` }; + } +} + /** * Preset provider list — embedded to avoid runtime YAML loading dependency. * Keep in sync with providers.yaml in cli-workflow. @@ -163,12 +199,16 @@ export async function cmdSetup(args: SetupArgs): Promise envData[envName] = args.apiKey; saveEnvFile(envPath, envData); + // Validate model connectivity + const validation = await validateModel(args.baseUrl, args.apiKey, args.model); + return { configPath, envPath, provider: args.provider, model: args.model, defaultAgent: merged.defaultAgent, + validation, }; } @@ -328,7 +368,7 @@ export async function cmdSetupInteractive(storageRoot: string): Promise Register a workflow"); console.log(' uwf thread start -p "..." Start a thread');