will
8e6ee31596
ci / test (push) Successful in 6m18s
History-free snapshot of the private development repository at cbe45c5 (2026-09-09). Code, docs and public catalogue only; no plant or job knowledge layers, no client or job names (tools/scrub.py clean). The intro deck is left out of this snapshot because its slide images could not be checked for job drawings.
174 lines
6.1 KiB
Python
174 lines
6.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Move long comment blocks out of a knowledge layer into a rationale file.
|
|
|
|
Values are never touched. Two gates prove it:
|
|
1. the parsed data must be byte-identical before and after;
|
|
2. every comment line that leaves the YAML must appear verbatim in the
|
|
rationale file.
|
|
Gate 1 is the caller's; gate 2 is here.
|
|
"""
|
|
import re, sys, json, hashlib
|
|
|
|
MIN_RUN = 10
|
|
KEY_RE = re.compile(r'^(\s*)([A-Za-z_][A-Za-z0-9_]*):\s*(.*)$')
|
|
BLOCK_RE = re.compile(r':\s*[|>][-+]?\d*\s*$')
|
|
SOURCE_RE = re.compile(r'#.*\bSOURCE\b')
|
|
# page / date / confidence tokens: these stay beside the value, so a number is
|
|
# never left in the file with its provenance a file away.
|
|
CITE_RE = re.compile(r'\bp\.?\s?(\d{2,4})\b|\b(\d{4}-\d\d-\d\d)\b'
|
|
r'|\b[Cc]onfidence ([A-D])\b')
|
|
|
|
|
|
def classify(lines):
|
|
"""'comment' | 'data' | 'blank' | 'scalar' for each line, block-aware.
|
|
|
|
A '#' inside a |/> block scalar is CONTENT, not a comment; treating it as
|
|
one would delete data.
|
|
"""
|
|
kinds = [None] * len(lines)
|
|
i = 0
|
|
while i < len(lines):
|
|
l = lines[i]
|
|
s = l.strip()
|
|
if BLOCK_RE.search(l) and not s.startswith('#'):
|
|
kinds[i] = 'data'
|
|
m = KEY_RE.match(l)
|
|
base = len(m.group(1)) if m else len(l) - len(l.lstrip())
|
|
i += 1
|
|
while i < len(lines):
|
|
nxt = lines[i]
|
|
if not nxt.strip():
|
|
kinds[i] = 'scalar'; i += 1; continue
|
|
if len(nxt) - len(nxt.lstrip()) <= base:
|
|
break
|
|
kinds[i] = 'scalar'; i += 1
|
|
continue
|
|
kinds[i] = 'blank' if not s else ('comment' if s.startswith('#') else 'data')
|
|
i += 1
|
|
return kinds
|
|
|
|
|
|
def path_of(lines, kinds, idx):
|
|
m = KEY_RE.match(lines[idx])
|
|
if not m:
|
|
return None
|
|
ind = len(m.group(1)); parts = [m.group(2)]
|
|
for j in range(idx - 1, -1, -1):
|
|
if kinds[j] != 'data':
|
|
continue
|
|
mj = KEY_RE.match(lines[j])
|
|
if not mj:
|
|
continue
|
|
ij = len(mj.group(1))
|
|
if ij < ind:
|
|
parts.append(mj.group(2)); ind = ij
|
|
if ind == 0:
|
|
break
|
|
return '.'.join(reversed(parts))
|
|
|
|
|
|
def run(src, out_yaml, out_md, title):
|
|
text = open(src, encoding='utf-8').read()
|
|
lines = text.split('\n')
|
|
kinds = classify(lines)
|
|
|
|
runs, cur = [], []
|
|
for i, k in enumerate(kinds):
|
|
if k == 'comment':
|
|
cur.append(i)
|
|
elif k == 'blank' and cur:
|
|
cur.append(i) # a blank line inside a block is a paragraph
|
|
else:
|
|
if cur: runs.append([j for j in cur if kinds[j] == 'comment'])
|
|
cur = []
|
|
if cur: runs.append([j for j in cur if kinds[j] == 'comment'])
|
|
runs = [r for r in runs if r]
|
|
|
|
moved = {}
|
|
for r in runs:
|
|
if len(r) < MIN_RUN or r[0] == 0: # file preamble stays put
|
|
continue
|
|
nxt = next((j for j in range(r[-1] + 1, len(lines)) if kinds[j] == 'data'), None)
|
|
if nxt is None:
|
|
continue
|
|
p = path_of(lines, kinds, nxt)
|
|
if p:
|
|
moved[r[0]] = [p, r]
|
|
|
|
seen = {}
|
|
for k in sorted(moved):
|
|
p, r = moved[k]
|
|
n = seen.get(p, 0); seen[p] = n + 1
|
|
if n:
|
|
moved[k][0] = '%s (%d)' % (p, n + 1)
|
|
|
|
md = ['# %s -- rationale' % title, '',
|
|
'Why the values in `knowledge.yaml` are what they are. Moved out of',
|
|
'that file so the keys the tool actually reads stay legible. NOTHING',
|
|
'HERE IS PARSED by anything -- it is for the reader, human or model.',
|
|
'',
|
|
'Each section is named for the key it explains. The key itself keeps a',
|
|
'`# sources:` stub and a pointer back here, so no number sits in the',
|
|
'layer with its provenance out of sight.',
|
|
'',
|
|
'Generated once by tools/layersplit.py; edit freely from here on.',
|
|
'', '---', '']
|
|
|
|
drop, inject, gate2 = set(), {}, []
|
|
for k in sorted(moved):
|
|
p, r = moved[k]
|
|
body, keep = [], []
|
|
pages, dates, conf = [], [], []
|
|
for j in r:
|
|
t = lines[j].strip()[1:]
|
|
if t.startswith(' '):
|
|
t = t[1:]
|
|
body.append(t)
|
|
gate2.append(t)
|
|
if SOURCE_RE.search(lines[j]):
|
|
keep.append(lines[j])
|
|
for pg, dt, cf in CITE_RE.findall(lines[j]):
|
|
if pg and 'p' + pg not in pages: pages.append('p' + pg)
|
|
if dt and dt not in dates: dates.append(dt)
|
|
if cf and cf not in conf: conf.append(cf)
|
|
md += ['## %s' % p, ''] + body + ['']
|
|
|
|
indent = re.match(r'^(\s*)', lines[r[0]]).group(1)
|
|
stub = []
|
|
if pages: stub.append(' '.join(pages[:6]) + (' +' if len(pages) > 6 else ''))
|
|
if dates: stub.append(' '.join(sorted(dates)[:3]) + (' +' if len(dates) > 3 else ''))
|
|
if conf: stub.append('confidence ' + '/'.join(sorted(conf)))
|
|
ptr = list(keep)
|
|
if stub:
|
|
ptr.append('%s# sources: %s' % (indent, '; '.join(stub)))
|
|
ptr.append('%s# -> rationale.md \u00a7 %s' % (indent, p))
|
|
inject[r[0]] = ptr
|
|
drop.update(r)
|
|
|
|
out = []
|
|
for i, l in enumerate(lines):
|
|
if i in inject:
|
|
out.extend(inject[i])
|
|
if i not in drop:
|
|
out.append(l)
|
|
|
|
yaml_txt = '\n'.join(out)
|
|
md_txt = '\n'.join(md)
|
|
open(out_yaml, 'w', encoding='utf-8').write(yaml_txt)
|
|
open(out_md, 'w', encoding='utf-8').write(md_txt)
|
|
|
|
# ---- gate 2: nothing was lost or mangled on the way out --------------
|
|
missing = [t for t in gate2 if t and t not in md_txt]
|
|
print('runs %d, moved %d' % (len(runs), len(moved)))
|
|
print('yaml %d -> %d lines (%.0f%% smaller)'
|
|
% (len(lines), len(out), 100.0 * (1 - len(out) / float(len(lines)))))
|
|
print('rationale %d lines' % len(md))
|
|
print('GATE 2 (every moved comment line present verbatim): %s'
|
|
% ('PASS' if not missing else 'FAIL -- %d missing' % len(missing)))
|
|
for t in missing[:5]:
|
|
print(' missing: %r' % t)
|
|
return 0 if not missing else 1
|
|
|
|
|
|
sys.exit(run(*sys.argv[1:]))
|