Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **fix(monitoring):** warn once when the stall sampler reports an unusable reading, reusing the existing sampler fallback ([#15978](https://github.com/diegosouzapw/OmniRoute/pull/15978)) — thanks @maxmad64bis
21 changes: 14 additions & 7 deletions src/lib/healthzLag.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,15 +78,22 @@ function resolveEventLoopStallThresholdMs(explicit?: number): number {
return parsed;
}

function readInflightSample(sampler: () => number): number {
function warnSamplerOnce(log: (message: string) => void): void {
if (stallWarnedSampler) return;
stallWarnedSampler = true;
log("[HEALTHZ] event loop stall inflight sampler unavailable, using 0");
}

function readInflightSample(sampler: () => number, log: (message: string) => void): number {
try {
const value = sampler();
return Number.isFinite(value) && value >= 0 ? Math.floor(value) : 0;
} catch {
if (!stallWarnedSampler) {
stallWarnedSampler = true;
console.warn("[HEALTHZ] event loop stall inflight sampler unavailable, using 0");
if (!Number.isFinite(value) || value < 0) {
warnSamplerOnce(log);
return 0;
}
return Math.floor(value);
} catch {
warnSamplerOnce(log);
return 0;
}
}
Expand Down Expand Up @@ -119,7 +126,7 @@ export function startEventLoopStallRecorder(opts: EventLoopStallRecorderOptions
stallMaxMs = Math.max(stallMaxMs, driftMs);
const seconds = Math.round(driftMs / 100) / 10;
const memory = process.memoryUsage();
const inflight = readInflightSample(sampler);
const inflight = readInflightSample(sampler, log);
log(
`[HEALTHZ] event loop stalled ${seconds}s ` +
`(rss=${Math.round(memory.rss)}, heapUsed=${Math.round(memory.heapUsed)}, ` +
Expand Down
102 changes: 94 additions & 8 deletions tests/unit/event-loop-stall.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -178,15 +178,64 @@ test("default clock measures ticks without throwing", () => {
}
});

test("invalid sampler reading falls back to zero and warns once", () => {
const { lines } = runStallScenario({
thresholdMs: 5000,
ticks: [0, 7000, 15000],
sampler: () => NaN,
});
const stallLines = lines.filter((line) => /inflight=\d/.test(line));
assert.equal(stallLines.length, 2);
assert.match(stallLines[0], /inflight=0/);
assert.match(stallLines[1], /inflight=0/);
const samplerWarnings = lines.filter((line) => line.includes("inflight sampler"));
assert.equal(samplerWarnings.length, 1);
});

test("negative sampler reading falls back to zero and warns once", () => {
const { lines } = runStallScenario({
thresholdMs: 5000,
ticks: [0, 7000, 15000],
sampler: () => -1,
});
const stallLines = lines.filter((line) => /inflight=\d/.test(line));
assert.equal(stallLines.length, 2);
assert.match(stallLines[0], /inflight=0/);
assert.match(stallLines[1], /inflight=0/);
const samplerWarnings = lines.filter((line) => line.includes("inflight sampler"));
assert.equal(samplerWarnings.length, 1);
});

test("unbounded sampler reading falls back to zero and warns once", () => {
const { lines } = runStallScenario({
thresholdMs: 5000,
ticks: [0, 7000, 15000],
sampler: () => Infinity,
});
const stallLines = lines.filter((line) => /inflight=\d/.test(line));
assert.equal(stallLines.length, 2);
assert.match(stallLines[0], /inflight=0/);
const samplerWarnings = lines.filter((line) => line.includes("inflight sampler"));
assert.equal(samplerWarnings.length, 1);
});

test("healthy sampler reading keeps its count without warning", () => {
const { lines } = runStallScenario({
thresholdMs: 5000,
ticks: [0, 7000, 15000],
sampler: () => 3,
});
const stallLines = lines.filter((line) => /inflight=\d/.test(line));
assert.equal(stallLines.length, 2);
assert.match(stallLines[0], /inflight=3/);
const samplerWarnings = lines.filter((line) => line.includes("inflight sampler"));
assert.equal(samplerWarnings.length, 0);
});

test("failing sampler falls back to zero and warns once", () => {
let now = 0;
let calls = 0;
const lines: string[] = [];
const warnings: string[] = [];
const originalWarn = console.warn;
console.warn = (message?: unknown, ...rest: unknown[]) => {
warnings.push([message, ...rest].map(String).join(" "));
};
try {
resetEventLoopStallStatsForTests();
startEventLoopStallRecorder({
Expand All @@ -205,10 +254,47 @@ test("failing sampler falls back to zero and warns once", () => {
now = 15000;
tickEventLoopStallRecorderForTests();
assert.equal(calls, 2);
assert.equal(lines.length, 2);
assert.match(lines[0], /inflight=0/);
const samplerWarnings = warnings.filter((line) => line.includes("inflight sampler"));
const stallLines = lines.filter((line) => /inflight=\d/.test(line));
assert.equal(stallLines.length, 2);
assert.match(stallLines[0], /inflight=0/);
assert.match(stallLines[1], /inflight=0/);
const samplerWarnings = lines.filter((line) => line.includes("inflight sampler"));
assert.equal(samplerWarnings.length, 1);
} finally {
stopEventLoopStallRecorder();
resetEventLoopStallStatsForTests();
}
});

test("invalid sampler reading warns through the injected log only", () => {
let now = 0;
const lines: string[] = [];
const consoleMessages: string[] = [];
const originalWarn = console.warn;
console.warn = (message?: unknown, ...rest: unknown[]) => {
consoleMessages.push([message, ...rest].map(String).join(" "));
};
try {
resetEventLoopStallStatsForTests();
startEventLoopStallRecorder({
thresholdMs: 5000,
now: () => now,
sampler: () => NaN,
log: (message: string) => {
lines.push(message);
},
});
for (const at of [0, 7000, 15000]) {
now = at;
tickEventLoopStallRecorderForTests();
}
const stallLines = lines.filter((line) => /inflight=\d/.test(line));
assert.equal(stallLines.length, 2);
assert.match(stallLines[0], /inflight=0/);
assert.match(stallLines[1], /inflight=0/);
const samplerWarnings = lines.filter((line) => line.includes("inflight sampler"));
assert.equal(samplerWarnings.length, 1);
assert.equal(consoleMessages.length, 0);
} finally {
console.warn = originalWarn;
stopEventLoopStallRecorder();
Expand Down
Loading