chore(ai-chat): guard-пороги в env + единый маркер дегенерации
Три хвоста детектора петель (#444): - Пороги детектора дегенерации теперь конфигурируются из env (по образцу AI_CHAT_FINAL_STEP_LOCKDOWN): AI_CHAT_DEGENERATION_REPEATED_LINES / _PERIOD_MAX_LEN / _PERIOD_MIN_REPEATS / _CHECK_STEP. Резолвер читает сырую строку (пусто = unset → компилируемый дефолт), требует ≥1, иначе безопасный откат к дефолту (0/отриц. сломал бы детектор). Оператор перенастраивает анти-babble-guard без редеплоя. - Единый маркер: при дегенерации live-стрим показывает нейтральное «Response stopped.» (клиент не отличает от ручного Stop), а персист-баннер после refetch падал в дженерик. Классифицировал OUTPUT_DEGENERATION_ERROR на клиенте под тот же заголовок «Response stopped.» + деталь про петлю — live и refetch больше не расходятся. - STEP_LIMIT_NO_ANSWER_MARKER: вместо хардкодной русской строки в content — локаль-нейтральная английская (базовая локаль = i18n-ключ этого репо, читаема моделью на реплее); не-русские юзеры больше не видят русский текст. Тесты: env-пороги гоняются против реальных форм повторов с mutation-проверкой (поднятый checkStep глушит burst; сниженный repeatedLines триггерит короткий ран); клиентская классификация — новый кейс в error-message.test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -89,6 +89,23 @@ describe("describeChatError", () => {
|
||||
expect(view.title).not.toBe("AI provider not configured");
|
||||
});
|
||||
|
||||
it("classifies a token-degeneration abort under the SAME 'Response stopped.' marker the live view shows (#495)", () => {
|
||||
// The exact reason the server persists in metadata.error on a degeneration
|
||||
// abort (ai-chat.service OUTPUT_DEGENERATION_ERROR). Live, this event shows
|
||||
// the neutral "Response stopped." notice; the persisted banner MUST match it
|
||||
// so live and refetch never disagree.
|
||||
const view = describeChatError(
|
||||
"Output degeneration detected (repeated token loop)",
|
||||
t,
|
||||
);
|
||||
expect(view.title).toBe("Response stopped.");
|
||||
expect(view.detail).toBe(
|
||||
"The answer was stopped automatically because the model fell into a repeated output loop.",
|
||||
);
|
||||
// Regression guard: it must NOT fall through to the generic heading.
|
||||
expect(view.title).not.toBe("Something went wrong");
|
||||
});
|
||||
|
||||
it("classifies a dropped connection (ECONNRESET) as a lost-connection error", () => {
|
||||
expect(
|
||||
describeChatError("Cannot connect to API: read ECONNRESET", t).title,
|
||||
|
||||
@@ -77,6 +77,22 @@ export function describeChatError(
|
||||
};
|
||||
}
|
||||
|
||||
// Our own token-degeneration abort (#444): the server aborts a runaway
|
||||
// repetition loop and persists this exact reason in metadata.error. LIVE, the
|
||||
// same abort surfaces as the neutral "Response stopped." notice (the client
|
||||
// cannot tell it from a manual Stop mid-stream), so the persisted banner must
|
||||
// read the SAME "Response stopped." marker — otherwise the live view and a
|
||||
// later refetch show two different texts for one event. The detail explains the
|
||||
// loop-guard cause without contradicting the shared heading.
|
||||
if (/output degeneration detected|repeated token loop/i.test(msg)) {
|
||||
return {
|
||||
title: t("Response stopped."),
|
||||
detail: t(
|
||||
"The answer was stopped automatically because the model fell into a repeated output loop.",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (/"statusCode"\s*:\s*403\b/.test(msg)) {
|
||||
return {
|
||||
title: t("AI chat is disabled"),
|
||||
|
||||
@@ -117,9 +117,14 @@ const FINAL_STEP_NUDGE =
|
||||
// NO text at all (#444, mitigates the "empty turn" the lockdown used to prevent
|
||||
// when the toggle is OFF). Makes the exhausted-without-answer state explicit to
|
||||
// the user and, on replay, to the model on the next turn.
|
||||
// The persisted content is the app's base locale (en-US) — which is ALSO the
|
||||
// i18n key the client localizes through `t()` — instead of a hardcoded Russian
|
||||
// string (it used to render Russian for every locale, and fed Russian back to
|
||||
// the model on replay). Keep it a plain, model-readable English sentence so the
|
||||
// next turn's replay reads cleanly; the client resolves the locale.
|
||||
const STEP_LIMIT_NO_ANSWER_MARKER =
|
||||
'(Достигнут лимит шагов — итоговый ответ не сформулирован; работа могла ' +
|
||||
'остаться незавершённой. Напишите «продолжай», чтобы агент продолжил.)';
|
||||
'(Step limit reached — no final answer was produced; the work may be ' +
|
||||
'unfinished. Reply "continue" to let the agent carry on.)';
|
||||
|
||||
// Reason recorded in ai_chat_runs.error / the assistant row when the token-
|
||||
// degeneration detector (#444) aborts a run. Distinct from a user Stop (no error)
|
||||
|
||||
@@ -9,9 +9,73 @@ import {
|
||||
DEGENERATION_CHECK_STEP,
|
||||
REPEATED_LINES_THRESHOLD,
|
||||
MIN_PERIOD_REPEATS,
|
||||
degenerationThresholds,
|
||||
} from './output-degeneration';
|
||||
import { AiChatService } from './ai-chat.service';
|
||||
|
||||
// Part A (#495 iter10): the detector thresholds are env-tunable. These drive the
|
||||
// resolver against real repeat-count shapes and mutation-verify that the env
|
||||
// override actually changes the trigger point (not a vacuous read).
|
||||
describe('degeneration thresholds are env-configurable', () => {
|
||||
const VARS = [
|
||||
'AI_CHAT_DEGENERATION_REPEATED_LINES',
|
||||
'AI_CHAT_DEGENERATION_PERIOD_MAX_LEN',
|
||||
'AI_CHAT_DEGENERATION_PERIOD_MIN_REPEATS',
|
||||
'AI_CHAT_DEGENERATION_CHECK_STEP',
|
||||
];
|
||||
const saved: Record<string, string | undefined> = {};
|
||||
beforeEach(() => {
|
||||
for (const v of VARS) saved[v] = process.env[v];
|
||||
});
|
||||
afterEach(() => {
|
||||
for (const v of VARS) {
|
||||
if (saved[v] === undefined) delete process.env[v];
|
||||
else process.env[v] = saved[v];
|
||||
}
|
||||
});
|
||||
|
||||
it('defaults to the compiled constants when unset', () => {
|
||||
for (const v of VARS) delete process.env[v];
|
||||
expect(degenerationThresholds()).toEqual({
|
||||
repeatedLines: REPEATED_LINES_THRESHOLD,
|
||||
maxPeriodLen: 150,
|
||||
minPeriodRepeats: MIN_PERIOD_REPEATS,
|
||||
checkStep: DEGENERATION_CHECK_STEP,
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to the default on blank / invalid / non-positive values', () => {
|
||||
for (const bad of ['', ' ', 'abc', '0', '-3', '1.5']) {
|
||||
process.env.AI_CHAT_DEGENERATION_REPEATED_LINES = bad;
|
||||
// '1.5' floors to 1 (still ≥1, valid); every other bad value → default.
|
||||
const expected = bad === '1.5' ? 1 : REPEATED_LINES_THRESHOLD;
|
||||
expect(degenerationThresholds().repeatedLines).toBe(expected);
|
||||
}
|
||||
});
|
||||
|
||||
it('a RAISED check-step suppresses a burst the default would have flagged', () => {
|
||||
// A ~3.3KB periodic burst is periodic-degenerate, but shouldCheckDegeneration
|
||||
// is the throttle gate. Default checkStep=2000 arms on it; raising the step
|
||||
// above the burst size means the throttle never re-fires for it.
|
||||
const burstLen = 'loadTools.\n'.repeat(300).length; // ~3300
|
||||
delete process.env.AI_CHAT_DEGENERATION_CHECK_STEP;
|
||||
expect(shouldCheckDegeneration(burstLen, 0)).toBe(true); // default 2000
|
||||
process.env.AI_CHAT_DEGENERATION_CHECK_STEP = String(burstLen + 1);
|
||||
expect(shouldCheckDegeneration(burstLen, 0)).toBe(false); // raised gate
|
||||
});
|
||||
|
||||
it('a LOWERED repeated-lines threshold trips on a shorter identical-line run', () => {
|
||||
// 8 identical lines: below the default 25 (rule 1) and below the periodic
|
||||
// rule's 20 repeats — so isDegenerateOutput is false by default.
|
||||
const shortRun = 'x\n'.repeat(8);
|
||||
delete process.env.AI_CHAT_DEGENERATION_REPEATED_LINES;
|
||||
expect(isDegenerateOutput(shortRun)).toBe(false);
|
||||
// Lower rule 1 to 5 → the 8-line run now trips.
|
||||
process.env.AI_CHAT_DEGENERATION_REPEATED_LINES = '5';
|
||||
expect(isDegenerateOutput(shortRun)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// Mock ONLY streamText so we can capture the onChunk/onStepFinish callbacks the
|
||||
// service registers and drive them by hand; every other `ai` export the service
|
||||
// uses (convertToModelMessages, stepCountIs, …) stays real.
|
||||
|
||||
@@ -23,6 +23,54 @@ export const MAX_PERIOD_LEN = 150;
|
||||
/** Rule 2: minimum number of consecutive block repeats to trigger. */
|
||||
export const MIN_PERIOD_REPEATS = 20;
|
||||
|
||||
/**
|
||||
* Read a positive-integer threshold from an env var, falling back to `fallback`
|
||||
* on unset/blank/invalid/non-positive. Mirrors the `AI_STREAM_PRE_RESPONSE_RETRIES`
|
||||
* resolver in `ai-streaming-fetch.ts`: read the RAW string first so a blank value
|
||||
* is treated as "unset" (→ fallback) rather than coercing to 0. Thresholds must
|
||||
* stay ≥ 1 — a 0/negative would make the detector fire on any text (or never), so
|
||||
* a bad value degrades to the safe compiled default instead. Env-tunable so an
|
||||
* operator can retune the anti-babble guard (#444) without a redeploy, following
|
||||
* the `AI_CHAT_FINAL_STEP_LOCKDOWN` toggle convention.
|
||||
*/
|
||||
function envThreshold(name: string, fallback: number): number {
|
||||
const rawStr = process.env[name];
|
||||
if (rawStr === undefined || rawStr.trim() === '') return fallback;
|
||||
const raw = Number(rawStr);
|
||||
return Number.isFinite(raw) && raw >= 1 ? Math.floor(raw) : fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the degeneration-detector thresholds from the environment, each
|
||||
* defaulting to the compiled constant above. Read fresh per call (not cached at
|
||||
* import) so a test — or a runtime env change — takes effect deterministically.
|
||||
*/
|
||||
export function degenerationThresholds(): {
|
||||
repeatedLines: number;
|
||||
maxPeriodLen: number;
|
||||
minPeriodRepeats: number;
|
||||
checkStep: number;
|
||||
} {
|
||||
return {
|
||||
repeatedLines: envThreshold(
|
||||
'AI_CHAT_DEGENERATION_REPEATED_LINES',
|
||||
REPEATED_LINES_THRESHOLD,
|
||||
),
|
||||
maxPeriodLen: envThreshold(
|
||||
'AI_CHAT_DEGENERATION_PERIOD_MAX_LEN',
|
||||
MAX_PERIOD_LEN,
|
||||
),
|
||||
minPeriodRepeats: envThreshold(
|
||||
'AI_CHAT_DEGENERATION_PERIOD_MIN_REPEATS',
|
||||
MIN_PERIOD_REPEATS,
|
||||
),
|
||||
checkStep: envThreshold(
|
||||
'AI_CHAT_DEGENERATION_CHECK_STEP',
|
||||
DEGENERATION_CHECK_STEP,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Rule 1 — ≥`REPEATED_LINES_THRESHOLD` consecutive IDENTICAL non-empty lines at
|
||||
* the tail. Catches the classic newline-delimited loop ("loadTools.\n" ×N).
|
||||
@@ -128,7 +176,11 @@ export function hasPeriodicTail(
|
||||
* Pure — the caller owns the abort side effect.
|
||||
*/
|
||||
export function isDegenerateOutput(text: string): boolean {
|
||||
return hasRepeatedLineRun(text) || hasPeriodicTail(text);
|
||||
const cfg = degenerationThresholds();
|
||||
return (
|
||||
hasRepeatedLineRun(text, cfg.repeatedLines) ||
|
||||
hasPeriodicTail(text, cfg.maxPeriodLen, cfg.minPeriodRepeats)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -154,7 +206,7 @@ export function shouldCheckDegeneration(
|
||||
textLen: number,
|
||||
lastCheckLen: number,
|
||||
): boolean {
|
||||
return textLen - lastCheckLen >= DEGENERATION_CHECK_STEP;
|
||||
return textLen - lastCheckLen >= degenerationThresholds().checkStep;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user