// $ per 1M tokens. cachedIn = 0.1x in, write = 1.25x in.
const PRICE = {
  "claude-opus-5":   { in: 5, out: 25, cachedIn: 0.50, write: 6.25 },
  "claude-sonnet-5": { in: 3, out: 15, cachedIn: 0.30, write: 3.75 },
  "claude-haiku-4-5":{ in: 1, out:  5, cachedIn: 0.10, write: 1.25 },
} as const;

export type Model = keyof typeof PRICE;

export function costOf(model: Model, u: Anthropic.Usage): number {
  const p = PRICE[model];
  return (
      u.input_tokens                       * p.in
    + (u.cache_read_input_tokens     ?? 0) * p.cachedIn
    + (u.cache_creation_input_tokens ?? 0) * p.write
    + u.output_tokens                      * p.out
  ) / 1_000_000;
}

// the two numbers that go in the runbook
const typical   = costOf("claude-opus-5", res.usage);
const worstCase = typical * MAX_AGENT_STEPS * (1 + MAX_RETRIES);

// One gate per dependency, sized from Little's Law.
export class Admission {
  private active = 0;
  private waiters: Array<() => void> = [];

  constructor(
    private maxConcurrent = 30,   // throughput x p95
    private maxQueued     = 60,   // past this: shed, don't stack
  ) {}

  async run<T>(fn: () => Promise<T>): Promise<T> {
    if (this.active >= this.maxConcurrent) {
      if (this.waiters.length >= this.maxQueued) {
        metrics.inc("admission.shed");
        throw new TooBusy({ retryAfterSec: 2 });  // a fast 429
      }
      metrics.gauge("admission.queue_depth", this.waiters.length);
      await new Promise<void>(r => this.waiters.push(r));
    }
    this.active++;
    try { return await fn(); }
    finally {
      this.active--;
      this.waiters.shift()?.();     // wake exactly one
    }
  }
}

export async function measured<T extends { usage: Anthropic.Usage }>(
  route: string,
  model: Model,
  call: () => Promise<T>,
): Promise<T> {
  const t0 = performance.now();
  try {
    const res = await call();
    const u = res.usage;
    log.info({
      route, model, ok: true,
      ms:        performance.now() - t0,
      costUsd:   costOf(model, u),
      tokensIn:  u.input_tokens,
      tokensOut: u.output_tokens,
      cacheRead: u.cache_read_input_tokens ?? 0,   // 0 forever = broken cache
    });
    return res;
  } catch (err) {
    log.error({ route, model, ok: false,
                ms: performance.now() - t0,
                kind: errorKind(err) });        // class, never message
    throw err;
  }
}

flowchart LR
  subgraph cacheable["Frozen prefix · bills at 0.1x"]
    direction TB
    T["1 · tool schemas"]:::froz --> S["2 · system prompt"]:::froz --> F["3 · few-shot examples"]:::froz
  end
  F --> BP{{"cache
breakpoint"}}:::bp
  BP --> V
  subgraph volatile["Volatile suffix · full price"]
    direction TB
    V["4 · retrieved chunks"]:::vol --> Q["5 · this turn's question"]:::vol
  end
  classDef froz fill:#D6F5E3,stroke:#1F2937,color:#0E1726;
  classDef vol fill:#FEE4E2,stroke:#1F2937,color:#0E1726;
  classDef bp fill:#FEF3C7,stroke:#1F2937,color:#0E1726;
          

const res = await client.messages.create({
  model: "claude-opus-5",
  max_tokens: 1024,
  system: [
    { type: "text", text: POLICY_DOC },              // ~40k, frozen
    { type: "text", text: FEW_SHOTS,
      cache_control: { type: "ephemeral", ttl: "1h" } },  // breakpoint
  ],
  messages: [{ role: "user", content: question }],       // volatile, after it
});

// prove it, on every deploy
const { cache_read_input_tokens: read = 0,
        cache_creation_input_tokens: wrote = 0 } = res.usage;
metrics.gauge("llm.cache.hit_ratio", read / (read + wrote + res.usage.input_tokens));

// silent invalidators, all of them in the prefix:
//   new Date()  ·  crypto.randomUUID()  ·  `user:${id}`
//   JSON.stringify(obj) with unordered keys  ·  a per-request tool list

const FLOOR  = 0.93;         // below this it's a different question
const TTL_MS = 15 * 60_000;   // "stale but fast" is not a feature

type Entry = { vec: Float32Array; answer: string; at: number };

export async function answerCached(
  tenant: string,                     // never share across tenants
  question: string,
  miss: () => Promise<string>,
) {
  const vec = await embed(question);
  const now = Date.now();

  for (const e of store.entries(tenant)) {
    if (now - e.at > TTL_MS) continue;
    if (cosine(vec, e.vec) >= FLOOR) {
      metrics.inc("cache.semantic.hit");
      return { answer: e.answer, hit: true };
    }
  }

  const answer = await miss();
  store.put(tenant, { vec, answer, at: now });
  return { answer, hit: false };
}

flowchart TB
  subgraph micro["1 · Micro-batch"]
    direction TB
    m0["+20ms, user still waiting"]:::hot --> m1["group 3 in-flight reqs"]:::hot --> m2["one embeddings call"]:::ok
  end
  subgraph api["2 · Batch API"]
    direction TB
    a0["async, ~50% off"]:::cold --> a1["submit 10k jobs"]:::cold --> a2["poll: minutes to hours"]:::ok
  end
  subgraph pack["3 · Prompt packing"]
    direction TB
    p0["cheapest, riskiest"]:::warn --> p1["10 items in one prompt"]:::cold --> p2["one shared context"]:::warn
  end
  classDef hot fill:#FEE4E2,stroke:#1F2937,color:#0E1726;
  classDef cold fill:#DCEBFE,stroke:#1F2937,color:#0E1726;
  classDef ok fill:#D6F5E3,stroke:#1F2937,color:#0E1726;
  classDef warn fill:#FEF3C7,stroke:#1F2937,color:#0E1726;
          

export class MicroBatcher<In, Out> {
  private q: { item: In; ok: (o: Out) => void; err: (e: unknown) => void }[] = [];
  private timer?: ReturnType<typeof setTimeout>;

  constructor(
    private run: (items: In[]) => Promise<Out[]>,
    private maxSize = 32,
    private maxWaitMs = 20,      // the latency you agree to add
  ) {}
  submit(item: In): Promise<Out> {
    return new Promise((ok, err) => {
      this.q.push({ item, ok, err });
      if (this.q.length >= this.maxSize) return this.flush();
      this.timer ??= setTimeout(() => this.flush(), this.maxWaitMs);
    });
  }

  private async flush() {
    clearTimeout(this.timer); this.timer = undefined;
    const batch = this.q.splice(0, this.maxSize);
    if (!batch.length) return;
    try {
      const out = await this.run(batch.map(b => b.item));
      batch.forEach((b, i) => b.ok(out[i]));
    } catch (e) {
      batch.forEach(b => b.err(e));      // never leave a caller hanging
    }
  }
}

sequenceDiagram
  autonumber
  participant U as User
  participant A as Your API
  participant M as Model
  U->>A: ask
  A->>M: stream: true
  M-->>A: first token (700ms)
  A-->>U: first word appears
  Note over U,A: user is now reading
  M-->>A: ...tokens...
  M-->>A: done (2100ms)
  A-->>U: complete
          

const t0 = performance.now();
let ttft: number | undefined;

const stream = client.messages.stream({
  model: "claude-sonnet-5",
  max_tokens: 1024,
  messages,
});

stream.on("text", (delta) => {
  ttft ??= performance.now() - t0;   // the number users feel
  res.write(delta);                    // straight out, no buffering
  res.flush?.();
});

const final = await stream.finalMessage();

if (final.stop_reason === "max_tokens") {
  log.warn({ msg: "truncated", route });     // half an answer shipped
}
log.info({
  ttftMs:  ttft,
  totalMs: performance.now() - t0,
  out:     final.usage.output_tokens,
  stop:    final.stop_reason,
});

res.writeHead(200, { "Content-Type": "text/event-stream",
                     "Cache-Control": "no-cache" });

let sent = 0;
try {
  const stream = client.messages.stream({
    model: "claude-sonnet-5", max_tokens: 1024, messages,
  });

  stream.on("text", (delta) => {
    sent += delta.length;
    res.write(`data: ${JSON.stringify({ delta })}\n\n`);
  });

  const final = await stream.finalMessage();
  res.write(`event: done\ndata: ${
    JSON.stringify({ stop: final.stop_reason })}\n\n`);

} catch (err) {
  // The status line left at byte 0. A 500 is no longer available.
  log.error({ msg: "mid-stream failure", sent, kind: errorKind(err) });
  res.write(`event: error\ndata: ${
    JSON.stringify({ retryable: sent === 0 })}\n\n`);
} finally {
  res.end();
}

const Draft = z.object({
  answer:     z.string(),
  confidence: z.number().min(0).max(1),   // ask, then act on it
});

async function cheap(q: string) {
  const r = await client.messages.parse({
    model: "claude-haiku-4-5",
    max_tokens: 800,
    output_config: { format: zodOutputFormat(Draft) },
    messages: [{ role: "user", content: q }],
  });
  return { ...r.parsed_output, stop: r.stop_reason };
}

export async function answer(q: string, plan: Plan) {
  if (plan === "enterprise" || needsTools(q)) return strong(q);

  const d = await cheap(q);
  const good = d.confidence >= 0.8
             && d.stop !== "max_tokens"
             && d.answer.length > 0;
  if (good) return d;

  metrics.inc("router.escalated");   // alert above ~30%:
  return strong(q);                    // you now pay for BOTH calls
}

const res = await client.messages.parse({
  model: "claude-haiku-4-5",
  max_tokens: 400,          // a ceiling the model cannot cross
  system: "Answer in at most three sentences. No preamble, "
        + "no restating the question, no closing summary.",
  output_config: {
    format: zodOutputFormat(Verdict),  // structure can't sprawl
    effort: "low",                     // thinking is a token bill too
  },
  stop_sequences: ["\n\nSources:"],   // stop at the useful part
  messages: [{ role: "user", content: q }],
});

// The tell: max_tokens means you ASKED for too much, not that
// the cap is too low. Shorten the ask before you raise the cap.
if (res.stop_reason === "max_tokens") {
  metrics.inc("answer.truncated");
}

// 3k in + 500 out on frontier = $0.0275.
// 3k in + 150 out        = $0.0188.  A 32% cut, no eval risk.

// The SDK retries 408/409/429/5xx + connection errors, with backoff.
// Configure it; do not reimplement it. (timeout is ms in TS.)
const client = new Anthropic({ maxRetries: 2, timeout: 20_000 });

// What you own: one deadline for the whole request, all attempts included.
export async function withDeadline<T>(
  ms: number,
  fn: (signal: AbortSignal) => Promise<T>,
): Promise<T> {
  const ac = new AbortController();
  const t = setTimeout(() => ac.abort(), ms);
  try { return await fn(ac.signal); }
  finally { clearTimeout(t); }
}

// And what counts as retryable, by class, never by message.
export function retryable(err: unknown): boolean {
  if (err instanceof Anthropic.RateLimitError)     return true;  // 429
  if (err instanceof Anthropic.APIConnectionError) return true;  // net
  if (err instanceof Anthropic.APIError) return (err.status ?? 0) >= 500;
  return false;   // 400 / 404: retrying just spends money slower
}

export async function hedged<T>(
  hedgeAfterMs: number,                  // your p95, not your mean
  call: (signal: AbortSignal) => Promise<T>,
): Promise<T> {
  const acs: AbortController[] = [];
  const attempt = () => {
    const ac = new AbortController(); acs.push(ac);
    return call(ac.signal);
  };

  const first = attempt();
  const hedge = new Promise<T>((resolve, reject) => {
    const t = setTimeout(() => {
      metrics.inc("hedge.fired");   // above ~10%? p95 moved
      attempt().then(resolve, reject);
    }, hedgeAfterMs);
    first.finally(() => clearTimeout(t));
  });

  try {
    return await Promise.race([first, hedge]);
  } finally {
    acs.forEach(ac => ac.abort());   // always cancel the loser
  }
}

stateDiagram-v2
  direction LR
  state "Half-open" as HalfOpen
  [*] --> Closed
  Closed --> Open: N failures in a row
  Open --> HalfOpen: cooldown elapsed
  HalfOpen --> Closed: probe succeeds
  HalfOpen --> Open: probe fails
          

type State = "closed" | "open" | "half-open";

export class Breaker {
  private state: State = "closed";
  private fails = 0;
  private openedAt = 0;
  constructor(
    private dep: string,              // one breaker per dependency
    private threshold = 5,
    private cooldownMs = 30_000,
  ) {}

  async run<T>(fn: () => Promise<T>, fallback: () => Promise<T>): Promise<T> {
    if (this.state === "open") {
      if (Date.now() - this.openedAt < this.cooldownMs) return fallback();
      this.state = "half-open";         // cooldown up: one probe
    }
    try {
      const out = await fn();
      this.fails = 0; this.state = "closed";
      return out;
    } catch {
      if (++this.fails >= this.threshold || this.state === "half-open") {
        this.state = "open"; this.openedAt = Date.now();
        log.error({ msg: "breaker opened", dep: this.dep });
      }
      return fallback();               // fail fast, then degrade
    }
  }
}

type Rung = { name: string; ms: number; run: (q: string) => Promise<Answer> };

const LADDER: Rung[] = [
  { name: "opus",  ms: 4000, run: q => call("claude-opus-5", q)   },
  { name: "haiku", ms: 1500, run: q => call("claude-haiku-4-5", q) },
  { name: "cache", ms:   50, run: q => nearestCachedAnswer(q)      },
  { name: "rules", ms:   10, run: q => cannedAnswer(q)             },
];

export async function answer(q: string): Promise<Answer> {
  for (const rung of LADDER) {
    try {
      const a = await withDeadline(rung.ms, () => rung.run(q));
      metrics.inc(`ladder.served.${rung.name}`);
      return { ...a, servedBy: rung.name };
    } catch (err) {
      metrics.inc(`ladder.fell.${rung.name}`);   // every fall is a metric
      log.warn({ rung: rung.name, kind: errorKind(err) });
    }
  }

  // the bottom rung is honest, not a crash
  return { text: "We couldn't answer that just now. Try again.",
           degraded: true, servedBy: "none" };
}

const Ticket = z.object({
  category: z.enum(["billing", "bug", "feature", "other"]),
  urgency:  z.number().int().min(1).max(5),
});

export async function triage(text: string, attempt = 0) {
  const r = await client.messages.parse({
    model: "claude-haiku-4-5",
    max_tokens: 256,
    output_config: { format: zodOutputFormat(Ticket) },
    messages: buildMessages(text, attempt),
  });

  // 1. why did it stop? check BEFORE touching content
  if (r.stop_reason === "max_tokens") return FALLBACK;  // half JSON
  if (r.stop_reason === "refusal")    return FALLBACK;  // empty

  // 2. does the shape hold?
  const parsed = Ticket.safeParse(r.parsed_output);
  if (parsed.success) return parsed.data;

  // 3. repair exactly once, then degrade. Never crash the request.
  if (attempt === 0) return triage(text, 1);
  metrics.inc("triage.unparseable");
  return FALLBACK;
}

// The model may retry. The refund must not.
export async function once<T>(key: string, effect: () => Promise<T>): Promise<T> {
  const claimed = await db.insertIfAbsent("effects", { key, state: "running" });
  if (!claimed) return db.awaitResult<T>(key);   // someone already did this

  try {
    const out = await effect();
    await db.complete(key, out);
    return out;
  } catch (err) {
    await db.fail(key);                        // releases the claim
    throw err;
  }
}

// derive the key from the INTENT: same ask = same key, forever
const key = hash(`${conversationId}:refund:${orderId}`);

await once(key, () =>
  stripe.refunds.create({ payment_intent }, { idempotencyKey: key }),
);

// NOT: hash(`refund:${Date.now()}`)  <- a new key on every attempt

export class Budget {
  private spentUsd = 0;
  private steps = 0;
  constructor(
    private capUsd   = 0.10,   // the worst case from slide 8
    private maxSteps = 8,
  ) {}

  /** Before every model call. Throws instead of quietly spending. */
  check() {
    if (this.steps >= this.maxSteps) throw new OverBudget("steps");
    if (this.spentUsd >= this.capUsd) throw new OverBudget("usd");
  }

  record(model: Model, u: Anthropic.Usage) {
    this.steps++;
    this.spentUsd += costOf(model, u);      // from slide 8
    metrics.gauge("budget.spent_usd", this.spentUsd);
  }
}

const budget = new Budget();
for (const step of agentLoop) {
  budget.check();                // fail INTO the ladder, not the bill
  const res = await measured(route, model, step);
  budget.record(model, res.usage);
}