#!/usr/bin/env node 'use strict'; /* * unreadable_split_audit.js v1.0.1 * 零依赖 Node 审计器:把 read_metaweb_pins_batch 回执里的「不可读」拆成 * rejected_by_requester(请求侧拒收:id 长度 ≠ 66,或 error 文本含 malformed) * unreadable_upstream (上游侧取不到:其余有错误文本者) * 并把 TIE(两侧信号冲突)与 N/A(无可判文本)单列,绝不洗进分母。 * * 用法(Node 外壳): * node unreadable_split_audit.js [--dir ] [--before ] [--json] * * 双跑设计:CORE-START / CORE-END 之间是纯函数(无 require、无 fs), * 浏览器版 unreadable_split_audit_page.html 会读取本文件、截取该段并原样执行, * 所以两处跑的是同一份判据器代码,不存在「页面另写一套」的漂移。 * * 退出码只由「审计器自身是否健康」决定(参数错误/目录不可读/自检失败), * 历史行(b 是否命中 161、某帧是否全错)永不驱动退出码。 */ /* ==== CORE-START ==== */ const VERSION = 'v1.0.1'; const TARGET_B = 161; // 上一枚件宣称的「每枚约 161 字节」 const TOL = 0.15; // ±15% const RECEIPT_RE = /^-\s+RECEIPT read_metaweb_pins_batch:\s*(\{.*\})\s*$/; const PIN_HEAD_RE = /^Pin\s+([0-9a-fA-F]+i\d+):\s*$/; const STATUS_HEAD_RE = /^##\s+([0-9a-fA-F]+i\d+)\s*$/; // ---------- 单帧解析 ---------- function parseFrame(lines, receiptLineIdx, segEnd, filePath) { const raw = RECEIPT_RE.exec(lines[receiptLineIdx]); let receipt = null, parseError = null; try { receipt = JSON.parse(raw[1]); } catch (e) { parseError = String((e && e.message) || e); } const statusBlocks = []; const readableIds = []; for (let j = receiptLineIdx + 1; j < segEnd; j++) { const line = lines[j]; const pm = PIN_HEAD_RE.exec(line); if (pm) { readableIds.push(pm[1]); continue; } const sm = STATUS_HEAD_RE.exec(line); if (!sm) continue; let k = j + 1; while (k < segEnd && lines[k].trim() === '') k++; if (k < segEnd && /^-\s+/.test(lines[k])) { const texts = []; let q = k; while (q < segEnd && /^-\s+/.test(lines[q])) { texts.push(lines[q].replace(/^-\s+/, '').trim()); q++; } statusBlocks.push({ id: sm[1], text: texts.join(' '), line: j + 1 }); j = q - 1; } } return { file: filePath, line: receiptLineIdx + 1, receipt, parseError, readableIds, statusBlocks }; } // ---------- 分类规则 ---------- // R(请求侧)信号:id 长度 ≠ 66,或文本含 malformed // U(上游侧)信号:存在文本且不含 malformed // R&U -> TIE;R only -> rejected;U only -> upstream;无文本且长度合规 -> N/A function classifyEntry(id, text) { const hasText = typeof text === 'string' && text.trim() !== ''; const lenOk = id.length === 66; const hasMalformed = hasText && /malformed/i.test(text); const rSignal = (!lenOk) || hasMalformed; const uSignal = hasText && !hasMalformed; if (rSignal && uSignal) return 'tie'; if (rSignal) return 'rejected_by_requester'; if (uSignal) return 'unreadable_upstream'; return 'na'; } // ---------- 帧归纳 ---------- function summarizeFrame(frame) { const rec = frame.receipt || {}; const declared = Number.isFinite(rec.unreadable) ? rec.unreadable : null; const listedIds = Array.isArray(rec.unreadable_ids) ? rec.unreadable_ids.slice() : []; const byId = new Map(); for (const b of frame.statusBlocks) { if (!byId.has(b.id)) byId.set(b.id, { id: b.id, text: b.text, blockLine: b.line }); } let listedOnly = 0; for (const id of listedIds) { if (!byId.has(id)) { byId.set(id, { id, text: '', blockLine: null }); listedOnly++; } } const counts = { rejected_by_requester: 0, unreadable_upstream: 0, tie: 0, na: 0 }; const detail = []; for (const entry of byId.values()) { const cls = classifyEntry(entry.id, entry.text); counts[cls]++; detail.push({ id: entry.id, cls, idLen: entry.id.length, text: entry.text || null, blockLine: entry.blockLine }); } return { file: frame.file, line: frame.line, requested: Number.isFinite(rec.requested) ? rec.requested : null, readable: Number.isFinite(rec.readable_upstream) ? rec.readable_upstream : null, returned: Number.isFinite(rec.returned) ? rec.returned : null, truncated: rec.truncated === true, declaredUnreadable: declared, listedIdCount: listedIds.length, enumerable: byId.size, listedOnly, counts, detail, bytesWritten: Number.isFinite(rec.bytes_written) ? rec.bytes_written : null, caliber: rec.caliber || null, parseError: frame.parseError, mtime: frame.mtime || null }; } // ---------- 线性拟合(按 N 分组取均值后 OLS) ---------- const T975 = { 1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228, 11: 2.201, 12: 2.179, 13: 2.160, 14: 2.145, 15: 2.131, 16: 2.120, 17: 2.110, 18: 2.101, 19: 2.093, 20: 2.086, 21: 2.080, 22: 2.074, 23: 2.069, 24: 2.064, 25: 2.060, 26: 2.056, 27: 2.052, 28: 2.048, 29: 2.045, 30: 2.042 }; function fitCost(frames) { const allError = frames.filter(f => f.requested !== null && f.readable === 0 && f.declaredUnreadable !== null && f.declaredUnreadable === f.requested && f.bytesWritten !== null ); const groups = new Map(); for (const f of allError) { if (!groups.has(f.requested)) groups.set(f.requested, []); groups.get(f.requested).push(f.bytesWritten); } const points = [...groups.entries()] .map(([n, ys]) => ({ n, k: ys.length, mean: ys.reduce((a, b) => a + b, 0) / ys.length })) .sort((a, b) => a.n - b.n); const result = { allErrorCount: allError.length, points, distinctN: points.length, status: 'undetermined', reason: '' }; if (points.length < 3) { result.reason = '不同 N 档少于 3 个(' + points.length + '),按纪律输出「未定」,不附 b 结论'; return result; } const n = points.length; const meanN = points.reduce((a, p) => a + p.n, 0) / n; const meanY = points.reduce((a, p) => a + p.mean, 0) / n; let sxx = 0, sxy = 0, syy = 0; for (const p of points) { sxx += (p.n - meanN) * (p.n - meanN); sxy += (p.n - meanN) * (p.mean - meanY); syy += (p.mean - meanY) * (p.mean - meanY); } const b = sxy / sxx; const a = meanY - b * meanN; let ssRes = 0; for (const p of points) { const pred = a + b * p.n; ssRes += (p.mean - pred) * (p.mean - pred); } const df = n - 2; const s2 = ssRes / df; const seB = Math.sqrt(s2 / sxx); const t = T975[df] || 1.96; const ciLow = b - t * seB; const ciHigh = b + t * seB; const r2 = syy > 0 ? 1 - ssRes / syy : null; const dev = Math.abs(b - TARGET_B) / TARGET_B; Object.assign(result, { a, b, seB, df, t, ciLow, ciHigh, r2, deviation: dev }); const ciExcludesZero = (ciLow > 0 && ciHigh > 0) || (ciLow < 0 && ciHigh < 0); if (!ciExcludesZero || dev > TOL) { result.status = 'void'; const why = []; if (!ciExcludesZero) why.push('b 的 95% 区间含 0'); if (dev > TOL) why.push('与 161 偏差 ' + (dev * 100).toFixed(1) + '% > 15%'); result.reason = why.join(';') + ' —— 在件内明写作废,只保留「失败帧非空」'; } else { result.status = 'pass'; result.reason = 'b 区间不含 0 且与 161 偏差 ' + (dev * 100).toFixed(1) + '% ≤ 15%'; } return result; } // ---------- 自检(正例 / 负例 / 控制 + TIE / N/A) ---------- function makeFixture(text) { const lines = text.split(/\r?\n/); const idx = lines.findIndex(l => RECEIPT_RE.test(l)); return parseFrame(lines, idx, lines.length, ''); } function runSelfTests() { const cases = []; const mk = (ch, hexLen) => ch.repeat(hexLen) + 'i0'; const bare = (ch, n) => ch.repeat(n); const idA = mk('a', 64), idB = mk('b', 64), idC = bare('c', 64), idR = mk('d', 64); const posText = [ '2/3 pin(s) readable in this batch:', '', '- RECEIPT read_metaweb_pins_batch: {"requested":4,"readable_upstream":1,"returned":1,"unreadable":3,"unreadable_ids":["' + idA + '","' + idB + '","' + idC + '"],"bytes_written":1000}', '', '## ' + idA, '- error: malformed pinId', '', '## ' + idB, '- error: pin not found', '', 'Pin ' + idR + ':', '- title: x' ].join('\n'); { const f = summarizeFrame(makeFixture(posText)); const ok = f.counts.rejected_by_requester === 2 && f.counts.unreadable_upstream === 1 && f.counts.tie === 0 && f.counts.na === 0; cases.push({ name: 'POS 正例(两列同时命中)', ok, got: JSON.stringify(f.counts), want: 'rejected=2 upstream=1 tie=0 na=0' }); } const idE = mk('e', 64); const negText = [ '- RECEIPT read_metaweb_pins_batch: {"requested":1,"readable_upstream":0,"returned":0,"unreadable":1,"unreadable_ids":["' + idE + '"],"bytes_written":500}', '', '## ' + idE, '- error: pin not found' ].join('\n'); { const f = summarizeFrame(makeFixture(negText)); const ok = f.counts.rejected_by_requester === 0 && f.counts.unreadable_upstream === 1; cases.push({ name: 'NEG 负例(合规 id 不得误判拒收)', ok, got: JSON.stringify(f.counts), want: 'rejected=0 upstream=1' }); } const ctlText = [ '- RECEIPT read_metaweb_pins_batch: {"requested":2,"readable_upstream":2,"returned":2,"unreadable":0,"unreadable_ids":[],"bytes_written":900}', '', 'Pin ' + mk('1', 64) + ':', '- title: a', '', 'Pin ' + mk('2', 64) + ':', '- title: b' ].join('\n'); { const f = summarizeFrame(makeFixture(ctlText)); const denom = f.counts.rejected_by_requester + f.counts.unreadable_upstream; const ok = denom === 0 && f.counts.tie === 0 && f.counts.na === 0 && f.enumerable === 0; cases.push({ name: 'CTL 控制(零不可读不产伪分类)', ok, got: 'denom=' + denom, want: 'denom=0' }); } const idF = mk('f', 62); // 62 hex + i0 = 64 字符:形状异常但可作状态块标题 const tieText = [ '- RECEIPT read_metaweb_pins_batch: {"requested":1,"readable_upstream":0,"returned":0,"unreadable":1,"unreadable_ids":["' + idF + '"],"bytes_written":300}', '', '## ' + idF, '- error: pin not found' ].join('\n'); { const f = summarizeFrame(makeFixture(tieText)); const ok = f.counts.tie === 1 && f.counts.rejected_by_requester === 0 && f.counts.unreadable_upstream === 0; cases.push({ name: 'TIE 冲突(两信号相反须单列)', ok, got: JSON.stringify(f.counts), want: 'tie=1 两列=0' }); } const idG = mk('9', 64); const naText = [ '- RECEIPT read_metaweb_pins_batch: {"requested":1,"readable_upstream":0,"returned":0,"unreadable":1,"unreadable_ids":["' + idG + '"],"bytes_written":200}' ].join('\n'); { const f = summarizeFrame(makeFixture(naText)); const denom = f.counts.rejected_by_requester + f.counts.unreadable_upstream; const ok = f.counts.na === 1 && denom === 0; cases.push({ name: 'N/A 无文本(单列且不进分母)', ok, got: 'na=' + f.counts.na + ' denom=' + denom, want: 'na=1 denom=0' }); } return cases; } // ---------- 报告(Node / 浏览器共用) ---------- function fmt(x, d) { return (x === null || x === undefined || Number.isNaN(x)) ? 'null' : Number(x).toFixed(d === undefined ? 2 : d); } function pct(x) { return (x === null || !Number.isFinite(x)) ? 'n/a' : (x * 100).toFixed(2) + '%'; } function buildReport(meta, frames) { const out = []; const totals = { rejected_by_requester: 0, unreadable_upstream: 0, tie: 0, na: 0 }; for (const fr of frames) for (const k of Object.keys(totals)) totals[k] += fr.counts[k]; const denom = totals.rejected_by_requester + totals.unreadable_upstream; const allErrorFrames = frames.filter(fr => fr.requested !== null && fr.readable === 0 && fr.declaredUnreadable !== null && fr.declaredUnreadable === fr.requested ); const fit = fitCost(frames); const selfTests = runSelfTests(); const dq = { truncatedIdList: frames.filter(fr => fr.declaredUnreadable !== null && fr.listedIdCount < fr.declaredUnreadable), lessEnum: frames.filter(fr => fr.declaredUnreadable !== null && fr.enumerable < fr.declaredUnreadable), moreEnum: frames.filter(fr => fr.declaredUnreadable !== null && fr.enumerable > fr.declaredUnreadable), inconsistent: frames.filter(fr => fr.requested !== null && fr.readable !== null && fr.declaredUnreadable !== null && fr.readable + fr.declaredUnreadable !== fr.requested), missingBytes: frames.filter(fr => fr.bytesWritten === null), parseErrors: frames.filter(fr => fr.parseError) }; out.push('=== 不可读分列审计器 ' + VERSION + ' ==='); out.push('runner : ' + meta.runner); out.push('corpus dir : ' + meta.dirLabel); out.push('freeze : ' + meta.freezeLabel); out.push(''); out.push('--- CENSUS ---'); out.push('扫描文件总数 : ' + meta.filesScanned); out.push('回执文件名命中数 : ' + meta.receiptFiles + (meta.excludedByBefore ? ('(冻结后纳入 ' + meta.includedFiles + ',排除 ' + meta.excludedByBefore + ')') : '')); out.push('回执帧数 : ' + frames.length); out.push('去重后唯一回执 : ' + (frames.length - meta.duplicates) + '(重复 ' + meta.duplicates + ')'); out.push('全错帧数 : ' + allErrorFrames.length); out.push('语料截止时刻(max mtime): ' + (meta.cutoffIso || 'null')); out.push('文件级错误 : ' + meta.fileErrors.length + (meta.fileErrors.length ? (' :: ' + meta.fileErrors.map(e => e.split(/[\\/]/).pop()).join(' | ')) : '')); out.push(''); out.push('--- 分列计数 ---'); out.push('列A rejected_by_requester : ' + totals.rejected_by_requester); out.push('列B unreadable_upstream : ' + totals.unreadable_upstream); out.push('TIE(单列,不进分母) : ' + totals.tie); out.push('N/A(单列,不进分母) : ' + totals.na); out.push('分母 = 列A + 列B : ' + denom); out.push('拒收占比(分母内) : ' + (denom > 0 ? pct(totals.rejected_by_requester / denom) : 'N/A(分母为 0,不产伪比率)')); out.push('「N/A 不洗进分母」 : ' + (denom === totals.rejected_by_requester + totals.unreadable_upstream ? ('已守住(N/A=' + totals.na + '、TIE=' + totals.tie + ' 均未计入 ' + denom + ')') : '未守住')); out.push(''); const withUnreadable = frames.filter(f => (f.declaredUnreadable || 0) > 0 || f.enumerable > 0); out.push('--- 逐帧明细(仅列有不可读项的帧;其余 ' + (frames.length - withUnreadable.length) + ' 帧两列均为 0) ---'); out.push('列: 文件:行 | N | 可读 | 声明不可读 | 可枚举 | 列A | 列B | TIE | N/A | bytes_written'); for (const fr of withUnreadable) { out.push(' ' + fr.file + ':' + fr.line + ' | ' + fr.requested + ' | ' + fr.readable + ' | ' + fr.declaredUnreadable + ' | ' + fr.enumerable + ' | ' + fr.counts.rejected_by_requester + ' | ' + fr.counts.unreadable_upstream + ' | ' + fr.counts.tie + ' | ' + fr.counts.na + ' | ' + fr.bytesWritten); for (const d of fr.detail) { out.push(' - ' + d.id + ' [' + d.cls + '] len=' + d.idLen + ' :: ' + (d.text || '(无状态文本)')); } } out.push('有不可读项的帧数: ' + withUnreadable.length); out.push(''); out.push('--- 失败帧成本复算(仅全错帧,按 N 分组取均值后 OLS: bytes_written = a + b·N) ---'); out.push('全错帧数: ' + fit.allErrorCount); out.push('按 N 分组: ' + (fit.points.length ? fit.points.map(p => 'N=' + p.n + '(k=' + p.k + ',mean=' + p.mean.toFixed(1) + ')').join(' ') : '(空)')); if (fit.status === 'undetermined') { out.push('结论: 未定 —— ' + fit.reason); out.push('不提 b 的点估计、不宣称 161,也不判作废(无数据可作废)。'); } else { out.push('a = ' + fmt(fit.a, 2) + ' 字节'); out.push('b = ' + fmt(fit.b, 2) + ' 字节/枚 95% CI [' + fmt(fit.ciLow, 2) + ', ' + fmt(fit.ciHigh, 2) + '] (df=' + fit.df + ', t=' + fit.t + ')'); out.push('R² = ' + (fit.r2 === null ? 'n/a' : fmt(fit.r2, 4)) + ' 与 161 偏差 = ' + pct(fit.deviation)); out.push('结论: ' + (fit.status === 'pass' ? 'PASS —— ' : 'VOID —— ') + fit.reason); } out.push(''); out.push('--- 自检(正例 / 负例 / 控制 / TIE / N/A) ---'); for (const c of selfTests) { out.push(' [' + (c.ok ? 'PASS' : 'FAIL') + '] ' + c.name + ' got: ' + c.got + ' want: ' + c.want); } out.push('自检结果: ' + selfTests.filter(c => c.ok).length + '/' + selfTests.length + (selfTests.every(c => c.ok) ? ' PASS' : ' FAIL')); out.push(''); out.push('--- 数据质量备注 ---'); out.push('unreadable_ids 数组短于声明不可读数: ' + (dq.truncatedIdList.map(f => f.file + ':' + f.line + '(声明 ' + f.declaredUnreadable + ',数组 ' + f.listedIdCount + ')').join(' | ') || '无')); out.push('可枚举 < 声明: ' + (dq.lessEnum.map(f => f.file + ':' + f.line + '(' + f.enumerable + '<' + f.declaredUnreadable + ')').join(' | ') || '无')); out.push('可枚举 > 声明: ' + (dq.moreEnum.map(f => f.file + ':' + f.line + '(' + f.enumerable + '>' + f.declaredUnreadable + ')').join(' | ') || '无')); out.push('可读+不可读 ≠ 请求: ' + (dq.inconsistent.map(f => f.file + ':' + f.line + '(' + f.readable + '+' + f.declaredUnreadable + '≠' + f.requested + ')').join(' | ') || '无')); out.push('缺 bytes_written 的帧: ' + dq.missingBytes.length); out.push('回执 JSON 解析失败: ' + dq.parseErrors.length); out.push(''); out.push('--- 退出码纪律 ---'); out.push('历史行(b 是否命中 161、是否有全错帧、某帧分类结果)不驱动退出码;'); out.push('退出码只反映审计器健康度:0=正常,2=参数/目录/自检失败。'); return { text: out.join('\n'), totals, denom, fit, selfTests, allErrorFrames, dq }; } function buildJson(meta, frames, report) { return { version: VERSION, runner: meta.runner, corpus: { dir: meta.dirLabel, before: meta.beforeIso || null, filesScanned: meta.filesScanned, receiptFiles: meta.receiptFiles, includedFiles: meta.includedFiles, excludedByBefore: meta.excludedByBefore, receipts: frames.length, duplicates: meta.duplicates, allErrorFrames: report.allErrorFrames.length, cutoff: meta.cutoffIso || null, fileErrors: meta.fileErrors }, totals: report.totals, denominator: report.denom, frames: frames.map(f => ({ file: f.file, line: f.line, requested: f.requested, readable: f.readable, declaredUnreadable: f.declaredUnreadable, enumerable: f.enumerable, counts: f.counts, bytesWritten: f.bytesWritten, detail: f.detail })), fit: report.fit, selfTests: report.selfTests }; } /* ==== CORE-END ==== */ /* ================= Node 外壳 ================= */ function parseArgs(argv) { const out = { dir: null, before: null, json: false, help: false }; for (let i = 2; i < argv.length; i++) { const a = argv[i]; if (a === '--dir') out.dir = argv[++i]; else if (a === '--before') out.before = argv[++i]; else if (a === '--json') out.json = true; else if (a === '--help' || a === '-h') out.help = true; else throw new Error('未知参数: ' + a); } return out; } function defaultSpillDir() { const os = require('os'); const appdata = process.env.APPDATA || require('path').join(os.homedir(), 'AppData', 'Roaming'); return require('path').join(appdata, 'IDBots', 'dsh-sessions', 'v0', 'spill'); } function walkFiles(dir, acc) { const fs = require('fs'); const path = require('path'); for (const ent of fs.readdirSync(dir, { withFileTypes: true })) { const p = path.join(dir, ent.name); if (ent.isDirectory()) walkFiles(p, acc); else if (ent.isFile()) acc.push(p); } return acc; } function main() { const fs = require('fs'); const path = require('path'); const args = parseArgs(process.argv); if (args.help) { console.log('用法: node unreadable_split_audit.js [--dir ] [--before ] [--json]'); process.exit(0); } const dir = args.dir || defaultSpillDir(); let before = null; if (args.before) { before = new Date(args.before); if (isNaN(before.getTime())) { console.error('[FATAL] --before 不是合法 ISO 时刻: ' + args.before); process.exit(2); } } let allFiles; try { allFiles = walkFiles(dir, []); } catch (e) { console.error('[FATAL] 目录不可读: ' + dir + ' :: ' + e.message); process.exit(2); } const receiptFiles = allFiles.filter(p => /read_metaweb_pins_batch/i.test(path.basename(p))); let excludedByBefore = 0; const included = []; for (const p of receiptFiles) { let st; try { st = fs.statSync(p); } catch (e) { continue; } if (before && st.mtime > before) { excludedByBefore++; continue; } included.push({ path: p, mtime: st.mtime }); } const frames = []; const fileErrors = []; for (const f of included) { let text; try { text = fs.readFileSync(f.path, 'utf8'); } catch (e) { fileErrors.push(f.path + ' :: ' + e.message); continue; } const lines = text.split(/\r?\n/); const idxs = []; for (let i = 0; i < lines.length; i++) if (RECEIPT_RE.test(lines[i])) idxs.push(i); if (idxs.length === 0) { fileErrors.push(f.path + ' :: 匹配到文件名但无 RECEIPT 行'); continue; } for (let k = 0; k < idxs.length; k++) { const start = idxs[k]; const end = (k + 1 < idxs.length) ? idxs[k + 1] : lines.length; const frame = parseFrame(lines, start, end, path.basename(f.path)); const s = summarizeFrame(frame); s.mtime = f.mtime.toISOString(); frames.push(s); } } let cutoff = null; for (const f of included) if (!cutoff || f.mtime > cutoff) cutoff = f.mtime; const hashSeen = new Map(); for (const fr of frames) { const key = JSON.stringify(fr); hashSeen.set(key, (hashSeen.get(key) || 0) + 1); } const duplicates = [...hashSeen.values()].filter(v => v > 1).reduce((a, v) => a + (v - 1), 0); const meta = { runner: 'node ' + process.version + ' / ' + process.platform + ' / ' + process.arch, dirLabel: dir, freezeLabel: before ? ('--before ' + before.toISOString()) : '未冻结(全量语料)', beforeIso: before ? before.toISOString() : null, filesScanned: allFiles.length, receiptFiles: receiptFiles.length, includedFiles: included.length, excludedByBefore, cutoffIso: cutoff ? cutoff.toISOString() : null, duplicates, fileErrors }; const report = buildReport(meta, frames); console.log(report.text); if (args.json) { const jsonPath = path.join(process.cwd(), 'unreadable_split_audit_result.json'); fs.writeFileSync(jsonPath, JSON.stringify(buildJson(meta, frames, report), null, 2), 'utf8'); console.log('\n[JSON] 已写出: ' + jsonPath); } process.exit(report.selfTests.every(c => c.ok) ? 0 : 2); } if (typeof require === 'function' && typeof process !== 'undefined' && process.argv) { main(); }