From 999018e3e030d6d5998f0889a870cbec3133602b Mon Sep 17 00:00:00 2001 From: Sunny Fung Date: Tue, 22 Sep 2026 11:25:34 +0800 Subject: [PATCH] fix: omni_read truncation declares first-N-of-M + cursor, stays under the shaping cap (H-71) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 238KB pins_by_path page used to come back as a 20,070 B spill file cut at char 16000 (mid-JSON, pagination cursor field gone) under a 'Full formatted result stored at:' banner: formatData cut at 20000 chars with a note pointing at the cursor field it had just removed, and the runtime tool-result shaper (dsh-runtime head+tail cut, 16000+4000) re-cut the over-cap result before spill-policy stored it. The tool output now stays at or under the shaper's 20000-char cap (truncation note budgeted out of MAX_RESULT_CHARS), so the spill file holds the complete, unshaped tool output and the banner is literally true. Truncation is declared honestly: first N of M chars, total M, and the echoed pagination token (cursor/nextCursor/lastId, top-level or under data) with a copy-pasteable re-run hint — or an explicit 'no pagination cursor' statement. pin_content gets the same first-N-of-M note for its raw body. --- src/main/libs/omniReaderAgentTools.ts | 48 +++++++++++++++++++++--- tests/omniReaderAgentTools.test.mjs | 53 +++++++++++++++++++++++++-- 2 files changed, 93 insertions(+), 8 deletions(-) diff --git a/src/main/libs/omniReaderAgentTools.ts b/src/main/libs/omniReaderAgentTools.ts index e47bfa32..9533b557 100644 --- a/src/main/libs/omniReaderAgentTools.ts +++ b/src/main/libs/omniReaderAgentTools.ts @@ -40,11 +40,47 @@ const MAN_BASE = 'https://man.metaid.io'; /** Keep large indexer payloads from flooding the conversation. */ const MAX_RESULT_CHARS = 20000; +// The dsh-runtime tool-result shaper (plugins/idbots-tool-result-shaping.mjs) +// head+tail-cuts any result over MAX_RESULT_CHARS — H-71: a 238KB indexer page +// came back as a spill file cut at char 16000 (mid-JSON, cursor field gone) +// under a "Full formatted result stored at:" banner. Reserve room for the +// truncation note so omni_read output always stays under that cap: the spill +// file then holds this tool's complete, honestly-declared output, unshaped. +const TRUNCATION_NOTE_RESERVE = 256; + +/** + * The pagination token echoed by the indexer, if any. manapi paged responses + * carry cursor/nextCursor at the top level or under `data`; show.now buzz + * lists echo lastId. A truncated page is only recoverable if the note names + * the token and its value — the old note pointed at a field it had just cut + * out of the JSON. + */ +function responseCursor(data: unknown): { key: string; value: string } | null { + if (data === null || typeof data !== 'object') return null; + const scopes: Array> = [data as Record]; + const nested = (data as Record).data; + if (nested !== null && typeof nested === 'object') scopes.push(nested as Record); + for (const scope of scopes) { + for (const key of ['cursor', 'nextCursor', 'lastId'] as const) { + const value = scope[key]; + if (typeof value === 'string' && value.trim()) return { key, value: value.trim() }; + } + } + return null; +} + +function truncationNote(kept: number, total: number, cursor: { key: string; value: string } | null): string { + if (cursor) { + return `[omni_read: first ${kept} of ${total} chars — ${cursor.key} "${cursor.value}" present; re-run with ${cursor.key}="${cursor.value}" for the next page]`; + } + return `[omni_read: first ${kept} of ${total} chars — no pagination cursor in the response; narrow the query (page/size) to reach the rest]`; +} function formatData(data: unknown): string { const text = typeof data === 'string' ? data : JSON.stringify(data, null, 2); - if (text.length > MAX_RESULT_CHARS) { - return `${truncateUtf16Units(text, MAX_RESULT_CHARS)}\n...(truncated, narrow the query with cursor/size)`; + if (text.length > MAX_RESULT_CHARS - TRUNCATION_NOTE_RESERVE) { + const kept = truncateUtf16Units(text, MAX_RESULT_CHARS - TRUNCATION_NOTE_RESERVE); + return `${kept}\n${truncationNote(kept.length, text.length, responseCursor(data))}`; } return text; } @@ -376,9 +412,11 @@ export function buildOmniReaderAgentTools(deps: { // Fire-and-forget chain-read ledger entry; the raw content body // has no metadata, so only pin id + text are recorded. recordChainReadSafe(readInputFromOmniJson(args.action, body, pinId, resolveMetabotId?.(sessionId ?? ''))); - const text = body.length > MAX_RESULT_CHARS - ? `${truncateUtf16Units(body, MAX_RESULT_CHARS)}\n...(truncated, narrow the query with cursor/size)` - : body; + let text = body; + if (body.length > MAX_RESULT_CHARS - TRUNCATION_NOTE_RESERVE) { + const kept = truncateUtf16Units(body, MAX_RESULT_CHARS - TRUNCATION_NOTE_RESERVE); + text = `${kept}\n[omni_read: first ${kept.length} of ${body.length} chars — raw content body; no pagination cursor]`; + } return textResult(text); } diff --git a/tests/omniReaderAgentTools.test.mjs b/tests/omniReaderAgentTools.test.mjs index be9780c5..4772a068 100644 --- a/tests/omniReaderAgentTools.test.mjs +++ b/tests/omniReaderAgentTools.test.mjs @@ -330,13 +330,60 @@ test('results are pretty-printed JSON', async () => { assert.equal(result.content[0].text, JSON.stringify({ code: 0, data: { name: 'alice' } }, null, 2)); }); -test('oversized results are truncated with a narrowing note', async () => { +test('oversized results carry an honest first-N-of-M note and stay under the 20k shaping cap', async () => { const big = { data: 'x'.repeat(30000) }; const { byName } = makeHarness({ fetchJsonResult: big }); const result = await byName.omni_read.handler({ action: 'indexer_stats' }); assert.equal(result.isError, undefined); - assert.match(result.content[0].text, /\(truncated, narrow the query with cursor\/size\)/); - assert.ok(result.content[0].text.length < 30000); + const text = result.content[0].text; + const total = JSON.stringify(big, null, 2).length; + // H-71: the runtime shaper (dsh-runtime/plugins/idbots-tool-result-shaping.mjs) + // head+tail-cuts anything over 20000 chars, so a bigger omni_read result used + // to spill as a mid-JSON-cut file under a "Full formatted result" banner. + // omni_read output must stay at or under that cap; the spill file then holds + // the complete declared output. + assert.ok(text.length <= 20000, `output stays under the shaping cap (got ${text.length})`); + assert.match(text, new RegExp(`\\[omni_read: first \\d+ of ${total} chars — `)); + assert.ok(text.includes('no pagination cursor in the response'), 'honestly reports the missing cursor'); + assert.ok(!text.includes('(truncated, narrow the query with cursor/size)'), 'the old misleading note is gone'); +}); + +test('truncation note surfaces the response pagination token so the page is recoverable', async () => { + // manapi shape: the cursor sits under data — exactly the field the old + // 20000-char head cut removed, leaving the note pointing at nothing. + const cursor = '4f3c9d1e77a2b608c5e4f19d2a8b7306c95e14f8d27a60b3c8e91f45d2076ab' + 'i0'; + const big = { + code: 0, + data: { list: [{ pin: { id: 'p1' }, summary: 'y'.repeat(30000) }], total: 53825, cursor }, + }; + const { byName } = makeHarness({ fetchJsonResult: big }); + const result = await byName.omni_read.handler({ action: 'pins_by_path', path: '/p', size: 100 }); + const text = result.content[0].text; + assert.ok(text.length <= 20000, `output stays under the shaping cap (got ${text.length})`); + assert.ok( + text.includes(`cursor "${cursor}" present; re-run with cursor="${cursor}" for the next page`), + 'note echoes the usable cursor value', + ); +}); + +test('truncation note also recognizes top-level nextCursor and nested lastId', async () => { + const topCursor = { code: 0, nextCursor: 'nc1', data: 'z'.repeat(30000) }; + const { byName } = makeHarness({ fetchJsonResult: topCursor }); + const top = await byName.omni_read.handler({ action: 'indexer_stats' }); + assert.ok(top.content[0].text.includes('nextCursor "nc1" present')); + + const lastId = { code: 0, data: { list: ['a'.repeat(30000)], lastId: 'L9' } }; + const { byName: byName2 } = makeHarness({ fetchJsonResult: lastId }); + const nested = await byName2.omni_read.handler({ action: 'buzz_hot', size: 10 }); + assert.ok(nested.content[0].text.includes('lastId "L9" present')); +}); + +test('pin_content truncation declares first-N-of-M on the raw body', async () => { + const { byName } = makeHarness({ fetchTextResult: 'b'.repeat(30000) }); + const result = await byName.omni_read.handler({ action: 'pin_content', pinId: 'txid1i0' }); + const text = result.content[0].text; + assert.ok(text.length <= 20000, `output stays under the shaping cap (got ${text.length})`); + assert.match(text, /\[omni_read: first \d+ of 30000 chars — raw content body; no pagination cursor\]/); }); test('fetch failures surface as an error result without throwing', async () => { base-commit: 2071265da0316f64a870c83b1327cccb32802801 -- 2.53.0