WillTools 2026-09-09 -- clean snapshot of the development repo for review
ci / test (push) Successful in 6m14s
ci / test (push) Successful in 6m14s
History-free snapshot of the private development repository at 77a82eb. Code, docs and the public catalogue only: no plant or job knowledge layers, no client, job or producer names (tools/scrub.py clean against the widened term list; a second word-level grep clean). The intro deck is left out because its slide images cannot be checked for job drawings.
This commit is contained in:
@@ -0,0 +1,173 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Move long comment blocks out of a knowledge layer into a rationale file.
|
||||
|
||||
Values are never touched. Two gates prove it:
|
||||
1. the parsed data must be byte-identical before and after;
|
||||
2. every comment line that leaves the YAML must appear verbatim in the
|
||||
rationale file.
|
||||
Gate 1 is the caller's; gate 2 is here.
|
||||
"""
|
||||
import re, sys, json, hashlib
|
||||
|
||||
MIN_RUN = 10
|
||||
KEY_RE = re.compile(r'^(\s*)([A-Za-z_][A-Za-z0-9_]*):\s*(.*)$')
|
||||
BLOCK_RE = re.compile(r':\s*[|>][-+]?\d*\s*$')
|
||||
SOURCE_RE = re.compile(r'#.*\bSOURCE\b')
|
||||
# page / date / confidence tokens: these stay beside the value, so a number is
|
||||
# never left in the file with its provenance a file away.
|
||||
CITE_RE = re.compile(r'\bp\.?\s?(\d{2,4})\b|\b(\d{4}-\d\d-\d\d)\b'
|
||||
r'|\b[Cc]onfidence ([A-D])\b')
|
||||
|
||||
|
||||
def classify(lines):
|
||||
"""'comment' | 'data' | 'blank' | 'scalar' for each line, block-aware.
|
||||
|
||||
A '#' inside a |/> block scalar is CONTENT, not a comment; treating it as
|
||||
one would delete data.
|
||||
"""
|
||||
kinds = [None] * len(lines)
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
l = lines[i]
|
||||
s = l.strip()
|
||||
if BLOCK_RE.search(l) and not s.startswith('#'):
|
||||
kinds[i] = 'data'
|
||||
m = KEY_RE.match(l)
|
||||
base = len(m.group(1)) if m else len(l) - len(l.lstrip())
|
||||
i += 1
|
||||
while i < len(lines):
|
||||
nxt = lines[i]
|
||||
if not nxt.strip():
|
||||
kinds[i] = 'scalar'; i += 1; continue
|
||||
if len(nxt) - len(nxt.lstrip()) <= base:
|
||||
break
|
||||
kinds[i] = 'scalar'; i += 1
|
||||
continue
|
||||
kinds[i] = 'blank' if not s else ('comment' if s.startswith('#') else 'data')
|
||||
i += 1
|
||||
return kinds
|
||||
|
||||
|
||||
def path_of(lines, kinds, idx):
|
||||
m = KEY_RE.match(lines[idx])
|
||||
if not m:
|
||||
return None
|
||||
ind = len(m.group(1)); parts = [m.group(2)]
|
||||
for j in range(idx - 1, -1, -1):
|
||||
if kinds[j] != 'data':
|
||||
continue
|
||||
mj = KEY_RE.match(lines[j])
|
||||
if not mj:
|
||||
continue
|
||||
ij = len(mj.group(1))
|
||||
if ij < ind:
|
||||
parts.append(mj.group(2)); ind = ij
|
||||
if ind == 0:
|
||||
break
|
||||
return '.'.join(reversed(parts))
|
||||
|
||||
|
||||
def run(src, out_yaml, out_md, title):
|
||||
text = open(src, encoding='utf-8').read()
|
||||
lines = text.split('\n')
|
||||
kinds = classify(lines)
|
||||
|
||||
runs, cur = [], []
|
||||
for i, k in enumerate(kinds):
|
||||
if k == 'comment':
|
||||
cur.append(i)
|
||||
elif k == 'blank' and cur:
|
||||
cur.append(i) # a blank line inside a block is a paragraph
|
||||
else:
|
||||
if cur: runs.append([j for j in cur if kinds[j] == 'comment'])
|
||||
cur = []
|
||||
if cur: runs.append([j for j in cur if kinds[j] == 'comment'])
|
||||
runs = [r for r in runs if r]
|
||||
|
||||
moved = {}
|
||||
for r in runs:
|
||||
if len(r) < MIN_RUN or r[0] == 0: # file preamble stays put
|
||||
continue
|
||||
nxt = next((j for j in range(r[-1] + 1, len(lines)) if kinds[j] == 'data'), None)
|
||||
if nxt is None:
|
||||
continue
|
||||
p = path_of(lines, kinds, nxt)
|
||||
if p:
|
||||
moved[r[0]] = [p, r]
|
||||
|
||||
seen = {}
|
||||
for k in sorted(moved):
|
||||
p, r = moved[k]
|
||||
n = seen.get(p, 0); seen[p] = n + 1
|
||||
if n:
|
||||
moved[k][0] = '%s (%d)' % (p, n + 1)
|
||||
|
||||
md = ['# %s -- rationale' % title, '',
|
||||
'Why the values in `knowledge.yaml` are what they are. Moved out of',
|
||||
'that file so the keys the tool actually reads stay legible. NOTHING',
|
||||
'HERE IS PARSED by anything -- it is for the reader, human or model.',
|
||||
'',
|
||||
'Each section is named for the key it explains. The key itself keeps a',
|
||||
'`# sources:` stub and a pointer back here, so no number sits in the',
|
||||
'layer with its provenance out of sight.',
|
||||
'',
|
||||
'Generated once by tools/layersplit.py; edit freely from here on.',
|
||||
'', '---', '']
|
||||
|
||||
drop, inject, gate2 = set(), {}, []
|
||||
for k in sorted(moved):
|
||||
p, r = moved[k]
|
||||
body, keep = [], []
|
||||
pages, dates, conf = [], [], []
|
||||
for j in r:
|
||||
t = lines[j].strip()[1:]
|
||||
if t.startswith(' '):
|
||||
t = t[1:]
|
||||
body.append(t)
|
||||
gate2.append(t)
|
||||
if SOURCE_RE.search(lines[j]):
|
||||
keep.append(lines[j])
|
||||
for pg, dt, cf in CITE_RE.findall(lines[j]):
|
||||
if pg and 'p' + pg not in pages: pages.append('p' + pg)
|
||||
if dt and dt not in dates: dates.append(dt)
|
||||
if cf and cf not in conf: conf.append(cf)
|
||||
md += ['## %s' % p, ''] + body + ['']
|
||||
|
||||
indent = re.match(r'^(\s*)', lines[r[0]]).group(1)
|
||||
stub = []
|
||||
if pages: stub.append(' '.join(pages[:6]) + (' +' if len(pages) > 6 else ''))
|
||||
if dates: stub.append(' '.join(sorted(dates)[:3]) + (' +' if len(dates) > 3 else ''))
|
||||
if conf: stub.append('confidence ' + '/'.join(sorted(conf)))
|
||||
ptr = list(keep)
|
||||
if stub:
|
||||
ptr.append('%s# sources: %s' % (indent, '; '.join(stub)))
|
||||
ptr.append('%s# -> rationale.md \u00a7 %s' % (indent, p))
|
||||
inject[r[0]] = ptr
|
||||
drop.update(r)
|
||||
|
||||
out = []
|
||||
for i, l in enumerate(lines):
|
||||
if i in inject:
|
||||
out.extend(inject[i])
|
||||
if i not in drop:
|
||||
out.append(l)
|
||||
|
||||
yaml_txt = '\n'.join(out)
|
||||
md_txt = '\n'.join(md)
|
||||
open(out_yaml, 'w', encoding='utf-8').write(yaml_txt)
|
||||
open(out_md, 'w', encoding='utf-8').write(md_txt)
|
||||
|
||||
# ---- gate 2: nothing was lost or mangled on the way out --------------
|
||||
missing = [t for t in gate2 if t and t not in md_txt]
|
||||
print('runs %d, moved %d' % (len(runs), len(moved)))
|
||||
print('yaml %d -> %d lines (%.0f%% smaller)'
|
||||
% (len(lines), len(out), 100.0 * (1 - len(out) / float(len(lines)))))
|
||||
print('rationale %d lines' % len(md))
|
||||
print('GATE 2 (every moved comment line present verbatim): %s'
|
||||
% ('PASS' if not missing else 'FAIL -- %d missing' % len(missing)))
|
||||
for t in missing[:5]:
|
||||
print(' missing: %r' % t)
|
||||
return 0 if not missing else 1
|
||||
|
||||
|
||||
sys.exit(run(*sys.argv[1:]))
|
||||
Reference in New Issue
Block a user