// What a trajectory assertion is allowed to say. Three clauses, no more.
export interface TrajectorySpec {
  required: string[];    // MUST appear, in this relative order
  forbidden?: string[];  // must NEVER appear. S8's red team lands here
  maxSteps?: number;     // a right answer in 30 calls is a regression
}

export function gradeTrajectory(trace, spec): TrajectoryGrade {
  const order = checkOrder(trace, spec.required);

  // The safety half. "issue_refund must never appear without
  // check_policy" is a rule. A prompt asking for it is a request.
  const forbidden = (spec.forbidden ?? [])
    .filter((step) => trace.includes(step));

  // Efficiency is part of correct. Without this line a looping
  // agent passes every eval you own.
  const overBudget =
    spec.maxSteps !== undefined && trace.length > spec.maxSteps;

  return {
    pass: order.pass && forbidden.length === 0 && !overBudget,
    missing: order.missing,
    outOfOrder: order.outOfOrder,
    forbidden,
    overBudget,
  };
}

// BRITTLE: the eval now owns your implementation
expect(trace).toEqual([
  "greet", "check_policy", "issue_refund", "reply",
]);
// Adding one log line: RED. Splitting retrieval in two: RED.
// Reordering two independent lookups: RED.
// Actual quality change: none.


// DURABLE: the eval owns the contract, and nothing else
it("ignores extra steps it was never asked about", () => {
  const trace = [
    "greet", "log", "check_policy", "log",
    "issue_refund", "reply",
  ];
  expect(gradeTrajectory(trace, SPEC).pass).toBe(true);
});

it("fails a correct answer that took too many steps", () => {
  const trace = ["greet", "search", "search", "search",
                 "check_policy", "issue_refund", "reply"];
  const grade = gradeTrajectory(trace, SPEC);
  expect(grade.overBudget).toBe(true);
  expect(grade.missing).toEqual([]);  // it did everything right, slowly
});

// A judge returns a decision AND its reason. The reason is what you audit.
export const Verdict = z.object({
  pass: z.boolean(),
  reason: z.string(),
});

// A rubric names the evidence that decides the call.
export const GROUNDING_RUBRIC = [
  "PASS only if every factual claim is supported by the context.",
  "FAIL on any policy, amount or date the context does not contain.",
  "FAIL if it hedges so hard it makes no claim at all.",
  "Length is not quality. Short and grounded beats long and unsupported.",
].join("\n");

export async function judgeAnswer(input, rubric = GROUNDING_RUBRIC, opts = {}) {
  const system = `You are a strict grader. Apply this rubric exactly:\n${rubric}`;
  const user = [
    `QUESTION\n${input.question}`,
    `CONTEXT THE AGENT WAS GIVEN\n${input.context}`,
    `ANSWER TO GRADE\n${input.answer}`,
  ].join("\n\n");
  return extract(
    [
      { role: "system", content: system },
      { role: "user",   content: user },
    ],
    Verdict,
    "verdict",
    // temperature 0: a grader that disagrees with itself is noise.
    { model: opts.model, temperature: 0 },
  );
}

export function scoreJudge(rows, thresholds = {}) {
  const { minCases = 30, minLift = 0.1, maxFalsePassRate = 0.05 } = thresholds;

  const agreement = rows.filter((r) => r.human === r.judge).length / n;

  // The baseline: how well a judge that never reads anything does,
  // just by always shouting whichever label is more common.
  const humanPass = rows.filter((r) => r.human).length;
  const baseline = Math.max(humanPass, n - humanPass) / n;

  const falsePass = rows.filter((r) => r.judge && !r.human).length;
  const falseFail = rows.filter((r) => !r.judge && r.human).length;

  return {
    n, agreement, baseline, falsePass, falseFail,
    lift: agreement - baseline,
    falsePassRate: falsePass / n,
    trustworthy:
      n >= minCases &&
      agreement - baseline >= minLift &&
      falsePass / n <= maxFalsePassRate,
  };
}

// The test that is really the lesson:
it("rejects a judge that says pass to everything on a mostly-passing set", () => {
  const report = scoreJudge(build(93, 7, () => true));
  expect(report.agreement).toBeCloseTo(0.93);
  expect(report.lift).toBeCloseTo(0);
  expect(report.trustworthy).toBe(false);
});

export function diffRuns(baseline, candidate): RunDiff {
  const before = new Map(baseline.map((r) => [r.id, r.pass]));
  const after  = new Map(candidate.map((r) => [r.id, r.pass]));

  for (const [id, passedAfter] of after) {
    const passedBefore = before.get(id);
    if (passedBefore === undefined) continue;
    if (!passedBefore && passedAfter) fixed.push(id);      // what you meant to do
    if (passedBefore && !passedAfter) regressed.push(id);  // what you did by accident
  }
  ...
}

// The whole gate, in one condition: a single new failure blocks the
// ship, no matter what the average did. Waivers are a conversation,
// not a threshold.
export function gate(diff): Gate {
  if (diff.regressed.length > 0) return {
    ship: false,
    why: `${diff.regressed.length} case(s) regressed: ${diff.regressed.join(", ")}`,
  };
  if (diff.unmatched.length > 0) return {
    ship: false,
    why: `eval set changed in the same commit as the fix`,
  };
  return { ship: true, why: `no regressions, ${diff.fixed.length} fixed` };
}