will
25592fa6c2
ci / test (push) Successful in 6m14s
History-free snapshot of the private development repository at 77a82eb. Code, docs and the public catalogue only: no plant or job knowledge layers, no client, job or producer names (tools/scrub.py clean against the widened term list; a second word-level grep clean). The intro deck is left out because its slide images cannot be checked for job drawings.
840 lines
39 KiB
Python
840 lines
39 KiB
Python
#!/usr/bin/env python3
|
|
"""Refuse to publish anything carrying a client's name.
|
|
|
|
python tools/scrub.py # working tree + full git history
|
|
python tools/scrub.py --worktree # working tree only, tracked and not
|
|
python tools/scrub.py --staged # what the pre-commit hook runs
|
|
python tools/scrub.py --private # the gitignored tree, on its own terms
|
|
|
|
Exits non-zero on a hit, so it can gate a commit, a package build, or a push.
|
|
|
|
--worktree and --staged look at different sets on purpose. --worktree is an
|
|
audit: every file git can see, tracked or not, and a hit anywhere is a hit.
|
|
--staged is a gate on one commit: only what is staged can block it, because
|
|
only what is staged is about to enter history. Untracked hits are still
|
|
printed, loudly, every time -- they are one `git add -A` from being the other
|
|
kind -- but they do not stop a commit they have nothing to do with. A gate
|
|
that blocks unrelated work is a gate people learn to pass with --no-verify.
|
|
|
|
--private is the third set, and it is an inventory rather than a gate: it
|
|
always exits 0. Every mode above honours .gitignore, so none of them has ever
|
|
opened `QNA/`, `knowledge/plant/` or `knowledge/job/` -- the other
|
|
repository's territory, where a name is legitimate, and also the largest body
|
|
of quoted source material in the checkout. Nothing looking at all is worse
|
|
than looking on the right terms, so --private looks, and asks a different
|
|
question: not "is there a name here", which is answered yes by design, but
|
|
"is there a name here in a shape somebody will copy" -- a fenced block, a
|
|
proposed key, a quoted literal, a path, a name glued into a longer
|
|
identifier. Those are aggregated by the distinct string and printed; names in
|
|
prose and in data values are counted and never listed. See `scan_private`.
|
|
|
|
It exits 0 because it has no verdict of its own to give. Every string it
|
|
prints contains either a denylist term or a structural pattern, so if one of
|
|
them had already been copied into a tracked file, --worktree would be failing
|
|
on that file right now. --private cannot find a leak that --worktree misses;
|
|
what it finds is the material a leak would be made out of, weeks before.
|
|
A mode that returned non-zero on that would return non-zero every single run,
|
|
and a check that always fails is a check that gets wrapped in `|| true`.
|
|
|
|
The terms to look for live in `.scrub-terms`, one per line, which is itself
|
|
gitignored -- the list of client names must never become the thing that leaks
|
|
them. `.scrub-terms.example` shows the format.
|
|
|
|
History matters as much as the working tree. Editing a file does not remove a
|
|
name from the commit that introduced it, and `git push` sends the commits, not
|
|
the checkout. A repo that has never been pushed can still be rewritten
|
|
cleanly; one that has been pushed cannot.
|
|
|
|
Structural patterns below are always checked, term list or not: a model
|
|
filename or an absolute user path identifies a job whether or not anyone
|
|
remembered to add it to the list.
|
|
|
|
The one exemption: the sanctioned placeholder names CLAUDE.md's "No client or
|
|
job information" table tells everyone to write instead of a real one -- the
|
|
job placeholder plus `.rvt` and friends -- are read out of that table (see
|
|
`load_placeholders`) and excused from the model/family-filename patterns,
|
|
whole-name only. Gluing anything else onto the placeholder still fails, on
|
|
purpose -- see `_is_placeholder_filename`.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
TERMS_FILE = os.path.join(ROOT, '.scrub-terms')
|
|
CLAUDE_MD = os.path.join(ROOT, 'CLAUDE.md')
|
|
# CLAUDE.md does not ship in a colleague distribution (tools/make_dist.py) --
|
|
# CONTRIBUTING.md is the fallback there, carrying the identical heading and
|
|
# table verbatim so `load_placeholders()` still finds it. Checked in this
|
|
# order; CLAUDE.md wins where both exist, which is every owner checkout.
|
|
CONTRIBUTING_MD = os.path.join(ROOT, 'CONTRIBUTING.md')
|
|
|
|
# Always suspicious, whatever the term list says.
|
|
STRUCTURAL = [
|
|
(r'[A-Za-z]:\\Users\\[A-Za-z0-9._-]+', 'absolute Windows user path'),
|
|
(r'\b[\w -]+\.rvt\b', 'Revit model filename'),
|
|
(r'\b[\w -]+\.rfa\b', 'Revit family filename'),
|
|
(r'\b[\w.+-]+@[\w-]+\.[\w.]+\b', 'email address'),
|
|
]
|
|
|
|
# The two structural categories above whose hit is a whole filename, and so
|
|
# can be checked against the sanctioned placeholders rather than a real name.
|
|
# The absolute-path and email patterns get no such exemption: there is no
|
|
# placeholder for those, and `sample_client`/`sample_job` never look like one.
|
|
FILENAME_CATEGORIES = {'Revit model filename', 'Revit family filename'}
|
|
|
|
# Files whose hits are noise rather than leaks.
|
|
SKIP = {'.scrub-terms', '.scrub-terms.example'}
|
|
SKIP_DIRS = {'.git', 'dist', '__pycache__', '_ticketqc'}
|
|
|
|
|
|
def load_terms(path=None) -> list[tuple[str, bool]]:
|
|
"""(term, case_sensitive) pairs. A leading '=' means case-sensitive.
|
|
|
|
Names are matched case-insensitively by default, because a client name is
|
|
a client name however it is typed. Office parameter names are the
|
|
exception: SCREAMING_SNAKE literals must not appear, but the lowercase
|
|
role identifiers that replaced them (panel.design_number) are the whole
|
|
point of the refactor and must not be flagged. Prefix those with '='.
|
|
"""
|
|
path = path or TERMS_FILE
|
|
if not os.path.isfile(path):
|
|
return []
|
|
terms = []
|
|
with open(path, encoding='utf-8') as fh:
|
|
for line in fh:
|
|
line = line.split('#', 1)[0].strip()
|
|
if not line:
|
|
continue
|
|
if line.startswith('='):
|
|
terms.append((line[1:].strip(), True))
|
|
else:
|
|
terms.append((line, False))
|
|
return [t for t in terms if t[0]]
|
|
|
|
|
|
def load_placeholders(path=None) -> list[str]:
|
|
"""The sanctioned stand-ins for a real client/job name -- read out of
|
|
CLAUDE.md's own table rather than retyped here.
|
|
|
|
CLAUDE.md Section 3 ("No client or job information in the repo. Ever.")
|
|
is the one place that defines what a placeholder is allowed to be:
|
|
|
|
| a client / owner | `sample_client` |
|
|
| a job / project / model | `sample_job` |
|
|
|
|
A second, hardcoded list in this file would drift the day someone adds or
|
|
renames a row there -- silently, because nothing would fail, the
|
|
exemption below would just stop covering the current placeholder. Reading
|
|
the table instead means there is exactly one place to edit and this file
|
|
always agrees with it.
|
|
|
|
Only the table between that heading and the following "Product codes
|
|
are..." paragraph is read, and only backtick-quoted tokens on table rows
|
|
(lines starting with "|") are taken -- the product-code examples two
|
|
paragraphs down (`PNL1.0`, `PNL101`, `ANCHOR_FACE_4T`) are prose, not a
|
|
table row, and are deliberately NOT placeholders: CLAUDE.md says outright
|
|
they "stay as they are", i.e. they must still be scrubbable text, not
|
|
exempted from anything.
|
|
|
|
If neither heading can be found -- both files missing, moved, or
|
|
reworded -- this returns [] rather than guessing at a fallback list. An
|
|
empty result only makes the filename check stricter (nothing is exempt),
|
|
never looser, which is the safe direction for a scrub gate to fail in.
|
|
|
|
NEITHER FILE, on a colleague distribution: CLAUDE.md is the owner's own
|
|
agent operating manual and does not ship (see tools/make_dist.py) --
|
|
CONTRIBUTING.md carries the identical heading and table there instead,
|
|
and is checked second when `path` is not given explicitly. Measured
|
|
2026-09-02: without this fallback, CI on a stripped checkout failed on
|
|
the sanctioned placeholder itself (`sample_job.rvt` in the test suite),
|
|
which is exactly the "nothing is exempt" failure mode above -- safe, but
|
|
a false positive on a totally clean fixture, not a leak.
|
|
"""
|
|
paths = [path] if path else [CLAUDE_MD, CONTRIBUTING_MD]
|
|
marker = 'No client or job information in the repo. Ever.'
|
|
for candidate in paths:
|
|
try:
|
|
with open(candidate, encoding='utf-8') as fh:
|
|
text = fh.read()
|
|
except OSError:
|
|
continue
|
|
start = text.find(marker)
|
|
if start == -1:
|
|
continue
|
|
end = text.find('Product codes are', start)
|
|
block = text[start:end] if end != -1 else text[start:start + 2000]
|
|
names = []
|
|
for line in block.splitlines():
|
|
line = line.strip()
|
|
if not line.startswith('|'):
|
|
continue
|
|
for m in re.finditer(r'`([^`]+)`', line):
|
|
names.append(m.group(1).strip().lower())
|
|
if names:
|
|
return names
|
|
return []
|
|
|
|
|
|
def _is_placeholder_filename(matched_text: str, placeholders: list[str]) -> bool:
|
|
"""True only if `matched_text` IS a placeholder name plus extension --
|
|
never if a placeholder merely appears somewhere inside it.
|
|
|
|
`os.path.splitext` peels off exactly the trailing `.rvt` / `.rfa`, and
|
|
the stem that is left is compared to the placeholder list whole. That
|
|
whole-name comparison is the point: a placeholder with anything else
|
|
appended to its stem is a DIFFERENT stem, absent from the list, so it is
|
|
still flagged. Only the placeholder alone, plus the extension, passes.
|
|
"""
|
|
if not placeholders:
|
|
return False
|
|
stem, ext = os.path.splitext(matched_text.strip())
|
|
if ext.lower() not in ('.rvt', '.rfa'):
|
|
return False
|
|
return stem.lower() in placeholders
|
|
|
|
|
|
|
|
# `\b` treats an underscore as a word character, the same as a letter, so it
|
|
# does not sit between them: a term does not match where it is immediately
|
|
# followed by "_SOMETHING" -- there is no boundary at the position between
|
|
# the term's last letter and the underscore. A term glued to a role
|
|
# identifier, a piece mark, or a version number this way slips the scan
|
|
# silently -- the exact shape of a name that is very much still in the file.
|
|
#
|
|
# What should end a match is a LETTER continuing the same word on either
|
|
# side -- that is the one case that means "this is a substring of a longer
|
|
# ordinary word, not the term" (a fake example: "Northgate" must not match
|
|
# inside "Northgateway" or "preNorthgate"). An underscore, a digit,
|
|
# punctuation, or the edge of the line all end the term just as surely as
|
|
# whitespace does, so none of those may appear immediately adjacent to the
|
|
# match either -- which is what these lookarounds check instead of `\b`.
|
|
_TERM_BOUNDARY_BEFORE = r'(?<![A-Za-z])'
|
|
_TERM_BOUNDARY_AFTER = r'(?![A-Za-z])'
|
|
|
|
|
|
def patterns(terms: list[tuple[str, bool]]) -> list[tuple[re.Pattern, str]]:
|
|
out = [(re.compile(p, re.I), why) for p, why in STRUCTURAL]
|
|
for term, cased in terms:
|
|
flags = 0 if cased else re.I
|
|
out.append((re.compile(_TERM_BOUNDARY_BEFORE + re.escape(term) +
|
|
_TERM_BOUNDARY_AFTER, flags),
|
|
'term {0!r}'.format(term)))
|
|
return out
|
|
|
|
|
|
def git(*args: str) -> str:
|
|
return subprocess.run(['git'] + list(args), cwd=ROOT, check=False,
|
|
capture_output=True, text=True,
|
|
errors='replace').stdout
|
|
|
|
|
|
def tracked_files() -> list[str]:
|
|
return [r for r in git('ls-files').splitlines() if r]
|
|
|
|
|
|
def untracked_files() -> list[str]:
|
|
"""Files git can see but is not tracking, .gitignore honoured.
|
|
|
|
These are where a name arrives: a scratch script somebody pasted a model
|
|
path into an hour ago is invisible to plain 'git ls-files', which is what
|
|
this scan used to run on. Ignored files stay out, so the real denylist and
|
|
the local job folders are not scanned.
|
|
"""
|
|
return [r for r in git('ls-files', '--others',
|
|
'--exclude-standard').splitlines() if r]
|
|
|
|
|
|
def staged_files() -> list[str]:
|
|
"""What this commit is actually about to write into history."""
|
|
out = git('diff', '--cached', '--name-only', '--diff-filter=ACMR')
|
|
return [r for r in out.splitlines() if r]
|
|
|
|
|
|
def scan_worktree(pats, listed=None, placeholders=None) -> list[str]:
|
|
hits = []
|
|
if listed is None:
|
|
listed = tracked_files() + untracked_files()
|
|
for rel in listed:
|
|
if os.path.basename(rel) in SKIP:
|
|
continue
|
|
path = os.path.join(ROOT, rel)
|
|
if not os.path.isfile(path):
|
|
continue
|
|
try:
|
|
with open(path, encoding='utf-8', errors='replace') as fh:
|
|
lines = fh.readlines()
|
|
except OSError:
|
|
continue
|
|
for n, line in enumerate(lines, 1):
|
|
for pat, why in pats:
|
|
m = pat.search(line)
|
|
if not m:
|
|
continue
|
|
if why in FILENAME_CATEGORIES and \
|
|
_is_placeholder_filename(m.group(0), placeholders):
|
|
continue
|
|
hits.append('{0}:{1}: {2} -- {3}'.format(
|
|
rel, n, why, m.group(0)[:60]))
|
|
return hits
|
|
|
|
|
|
def scan_history(pats, placeholders=None) -> list[str]:
|
|
"""Every version of every file that has ever been committed."""
|
|
if not git('rev-parse', '--git-dir').strip():
|
|
return []
|
|
log = git('log', '--all', '--patch', '--no-color', '--format=%H%x00%s')
|
|
hits, commit, subject = [], '?', '?'
|
|
seen = set()
|
|
for line in log.splitlines():
|
|
if '\x00' in line and len(line.split('\x00')[0]) == 40:
|
|
commit, subject = line.split('\x00', 1)
|
|
continue
|
|
if not line.startswith('+'):
|
|
continue
|
|
for pat, why in pats:
|
|
m = pat.search(line)
|
|
if not m:
|
|
continue
|
|
if why in FILENAME_CATEGORIES and \
|
|
_is_placeholder_filename(m.group(0), placeholders):
|
|
continue
|
|
key = (commit, why, m.group(0))
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
hits.append('{0} {1!r}: {2} -- {3}'.format(
|
|
commit[:8], subject[:40], why, m.group(0)[:60]))
|
|
return hits
|
|
|
|
|
|
# ---------------------------------------------------------------- private tree
|
|
#
|
|
# Everything above honours .gitignore, on purpose and for good reasons -- and
|
|
# the consequence is that nothing in this file has ever opened `QNA/`,
|
|
# `knowledge/plant/` or `knowledge/job/`. Those are the other repository's
|
|
# territory (see the .gitignore comment that says so), a name is legitimate
|
|
# in them, and they are also the largest body of prose in this working tree
|
|
# and the only part of it whose job is to quote source material verbatim. A
|
|
# name is not merely possible there, it is certain.
|
|
#
|
|
# So this is deliberately NOT a gate and must never be grown into one. If it
|
|
# flagged every producer name in a plant file it would print a wall of
|
|
# expected hits, and a wall of expected hits gets read exactly once. That is
|
|
# the same failure --staged avoids by not blocking a commit on untracked
|
|
# files, one directory further out.
|
|
#
|
|
# The question --private asks is therefore not "is there a name in here" --
|
|
# answered yes, obviously, that is what the file is -- but "is there a name
|
|
# in here in a shape that somebody will COPY". A sentence of prose about a
|
|
# producer's practice stays in the private repo when the note is read. A
|
|
# fenced block, a proposed YAML key, a quoted literal, a path: those are the
|
|
# strings that get lifted out of an extraction note and pasted into a module,
|
|
# a docs page or a test, and they cross the repo boundary with the name still
|
|
# inside them. That is the distinction this mode is built on, and it is what
|
|
# keeps the report short enough to still be read next month. (`_scratch/` and
|
|
# `sample data/` are gitignored too and get the same treatment, for the same
|
|
# reason: nothing was looking at them either.)
|
|
#
|
|
# Be honest about what it cannot do, because the shape of the blind spot is
|
|
# the useful part:
|
|
#
|
|
# * It cannot judge prose. A name in a sentence is counted, never read, so
|
|
# a producer's name inside a verbatim quotation that somebody later
|
|
# pastes whole into a docs page is resident here and invisible. What
|
|
# catches that is --worktree, on the day it lands in a tracked file.
|
|
# * It cannot read a PDF, and the exported tickets carry the job name in
|
|
# the title block. Those are in the "not read" tally, deliberately
|
|
# visible as a number.
|
|
# * "Travelling" is a guess about a shape, not a proof about a future.
|
|
# `key: value` in prose and a Markdown fence that is really a diagram
|
|
# both look like code to it, and a name written into a sentence that is
|
|
# about to become a docstring does not.
|
|
# * It only knows names that are on the term list, plus the structural
|
|
# patterns. A producer nobody has added to `.scrub-terms` is as
|
|
# invisible here as it is everywhere else.
|
|
|
|
PRIVATE_SKIP_DIRS = {'__pycache__', 'dist', 'build', 'node_modules',
|
|
'site-packages'}
|
|
|
|
# Opaque to a line scanner: binary, or an export whose text this cannot read
|
|
# anyway. They are counted and reported as a number rather than passed over
|
|
# silently -- "not scanned" and "scanned, clean" are different answers and a
|
|
# scrub report may not collapse them. A ticket PDF is the standing example:
|
|
# it carries the job name in its title block and nothing here can see it.
|
|
PRIVATE_SKIP_EXT = {'.pdf', '.rvt', '.rfa', '.png', '.jpg', '.jpeg', '.gif',
|
|
'.bmp', '.ico', '.zip', '.7z', '.gz', '.tar', '.xlsx',
|
|
'.xls', '.docx', '.doc', '.pptx', '.pyc', '.pyo', '.dll',
|
|
'.exe', '.so', '.dat', '.db', '.sqlite', '.bin'}
|
|
|
|
PRIVATE_MAX_BYTES = 2000000
|
|
|
|
# What kind of file this is decides which contexts count as travelling below.
|
|
PRIVATE_DOC_EXT = {'', '.md', '.markdown', '.txt', '.text', '.rst'}
|
|
PRIVATE_DATA_EXT = {'.yaml', '.yml', '.json', '.csv', '.tsv', '.toml', '.ini',
|
|
'.cfg', '.xml'}
|
|
|
|
_FENCE = re.compile(r'^\s{0,3}(```|~~~)')
|
|
_INLINE_CODE = re.compile(r'`[^`]+`')
|
|
_QUOTED = re.compile(r"'[^']*'|\"[^\"]*\"")
|
|
# A YAML-shaped line: a space-free lowercase key, a colon, a value. Prose
|
|
# rarely looks like this -- "Note: the producer says" is capitalised and
|
|
# "the plant says: x" has a space in the key, and the key pattern excludes
|
|
# both. What this is for is the proposed shape written INTO a note --
|
|
# `plant_default: 4` in the middle of a paragraph -- which is a line somebody
|
|
# is about to paste into knowledge.DEFAULTS.
|
|
_YAML_LINE = re.compile(r'^\s*(-\s+)?[a-z_][a-z0-9_.\-]*:\s*\S')
|
|
# A whitespace-and-punctuation-delimited token: the unit a name gets glued
|
|
# into, and the unit somebody copies. Angle brackets and `=` are delimiters
|
|
# too, or a name inside a generated HTML report comes back as sixty
|
|
# characters of markup with the interesting part buried in the middle.
|
|
_TOKEN = re.compile(r'[^\s`\'"(),;<>=]+')
|
|
|
|
|
|
def ignored_files() -> list[str]:
|
|
"""Files git can see and is deliberately ignoring.
|
|
|
|
The mirror image of `untracked_files()`: the same walk, the other side of
|
|
.gitignore. Between this, `untracked_files()` and `tracked_files()` there
|
|
is no longer any part of the working tree that no mode ever opens.
|
|
"""
|
|
out = git('ls-files', '--others', '--ignored', '--exclude-standard')
|
|
return [r for r in out.splitlines() if r]
|
|
|
|
|
|
def private_files(listed=None):
|
|
"""The gitignored files worth reading, and a tally of what was dropped.
|
|
|
|
The tally is returned rather than discarded because the count is part of
|
|
the answer. A report that says "480 files read" while silently passing
|
|
over 200 PDFs is claiming a coverage it does not have -- the same reason
|
|
a check that could not run may not render as OK.
|
|
|
|
Dot-prefixed path segments go in the tally, not the scan: one agent's
|
|
browser profile cache under `QNA/` is 58,000 files on its own, and
|
|
dot-names in a working tree are tool state rather than documents.
|
|
"""
|
|
if listed is None:
|
|
listed = ignored_files()
|
|
keep = []
|
|
dropped = {'tool state': 0, 'build output': 0, 'unreadable': 0,
|
|
'over size cap': 0, 'gone': 0}
|
|
for rel in listed:
|
|
parts = rel.replace('\\', '/').split('/')
|
|
if any(p.startswith('.') for p in parts):
|
|
dropped['tool state'] += 1
|
|
continue
|
|
if os.path.basename(rel) in SKIP:
|
|
dropped['tool state'] += 1
|
|
continue
|
|
if any(p in PRIVATE_SKIP_DIRS for p in parts):
|
|
dropped['build output'] += 1
|
|
continue
|
|
if os.path.splitext(rel)[1].lower() in PRIVATE_SKIP_EXT:
|
|
dropped['unreadable'] += 1
|
|
continue
|
|
try:
|
|
size = os.path.getsize(os.path.join(ROOT, rel))
|
|
except OSError:
|
|
dropped['gone'] += 1
|
|
continue
|
|
if size > PRIVATE_MAX_BYTES:
|
|
dropped['over size cap'] += 1
|
|
continue
|
|
keep.append(rel)
|
|
return keep, dropped
|
|
|
|
|
|
def _file_kind(rel: str) -> str:
|
|
ext = os.path.splitext(rel)[1].lower()
|
|
if ext in PRIVATE_DOC_EXT:
|
|
return 'doc'
|
|
if ext in PRIVATE_DATA_EXT:
|
|
return 'data'
|
|
return 'code'
|
|
|
|
|
|
def _token_at(line: str, pos: int) -> str:
|
|
for tok in _TOKEN.finditer(line):
|
|
if tok.start() <= pos < tok.end():
|
|
return tok.group(0)
|
|
return ''
|
|
|
|
|
|
def _inside(pos: int, spans) -> bool:
|
|
"""True if `pos` is strictly inside one of these delimited spans, i.e.
|
|
between the delimiters rather than on one."""
|
|
return any(s.start() < pos < s.end() - 1 for s in spans)
|
|
|
|
|
|
def _travels(kind: str, line: str, start: int, why: str, in_fence: bool):
|
|
"""Where this hit sits, when that place is one a string travels out of --
|
|
otherwise None, meaning the name is *resident*: this file's own subject
|
|
matter, sitting in the repository where it is allowed to be.
|
|
|
|
Read the branches as a claim about copy-and-paste, not about danger.
|
|
Nothing here is unsafe where it is. The only question is which of these
|
|
strings has a plausible future in a tracked file.
|
|
"""
|
|
if not why.startswith('term '):
|
|
# Structural. A machine's user path, an email address, a model or
|
|
# family filename: none of those is a plant convention or a
|
|
# producer's vocabulary. Each identifies one person, one machine or
|
|
# one job, and reads identically wherever it is pasted, so there is
|
|
# no "doing its job" reading of it to weigh against travelling.
|
|
return 'structural'
|
|
tok = _token_at(line, start)
|
|
if '/' in tok or '\\' in tok:
|
|
return 'in a path'
|
|
if kind == 'code':
|
|
# A private script is all instruction. There is no prose half of it
|
|
# for a name to be resident in, and a scratch script is the exact
|
|
# thing the untracked-file scan already exists for -- this one just
|
|
# happens to sit under an ignored directory.
|
|
return 'in code'
|
|
if kind == 'data':
|
|
# Values in a plant or job file ARE the file. A comment is not: it
|
|
# is written for a reader, in sentences, and it is what gets carried
|
|
# across when the shape of the file is proposed somewhere else.
|
|
hash_at = line.find('#')
|
|
if hash_at != -1 and hash_at < start:
|
|
return 'in a comment'
|
|
return None
|
|
if in_fence:
|
|
return 'in a fenced code block'
|
|
if _inside(start, _INLINE_CODE.finditer(line)):
|
|
return 'in inline code'
|
|
if _inside(start, _QUOTED.finditer(line)):
|
|
return 'in a quoted literal'
|
|
if _YAML_LINE.match(line):
|
|
return 'in a key: value line'
|
|
return None
|
|
|
|
|
|
def _safe(text: str) -> str:
|
|
"""Whatever of `text` this console can actually print.
|
|
|
|
The private tree is notes, and notes are written in whatever language the
|
|
note-taker was thinking in. A Windows console defaulting to cp1252 raises
|
|
UnicodeEncodeError on the first character it cannot render, which kills
|
|
the process halfway down the report -- a scrub that stops printing looks
|
|
exactly like a scrub that found nothing more.
|
|
"""
|
|
enc = getattr(sys.stdout, 'encoding', None) or 'ascii'
|
|
return text.encode(enc, 'replace').decode(enc, 'replace')
|
|
|
|
|
|
def scan_private(pats, listed=None, placeholders=None):
|
|
"""Read the private tree and sort what is found into three piles.
|
|
|
|
Returns `(travelling, named_dirs, resident)`:
|
|
|
|
travelling {token: [count, example, why, glued, where]} -- names in
|
|
a shape that gets copied. Keyed on the whole delimited
|
|
TOKEN the name sits in, not on the line, because the unit
|
|
that travels is a string, not an occurrence: forty
|
|
mentions of one parameter name are one thing to decide
|
|
about, and forty lines of it is the wall this mode exists
|
|
to avoid printing. `glued` is True when the token is
|
|
larger than the name itself -- the name buried inside an
|
|
identifier, a filename, a piece mark. That is the shape a
|
|
human reader skims past, so it sorts to the top.
|
|
named_dirs {(why, text): [count, example]} -- names in the tree's own
|
|
file and folder names, aggregated by the name. A job
|
|
folder named after the job produces one of these for every
|
|
file underneath it; 380 identical lines would drown
|
|
everything above.
|
|
resident {path: count} -- everything else, reported as numbers.
|
|
Never listed. A producer's name in a sentence about that
|
|
producer, in the repository that documents them, is the
|
|
file working correctly.
|
|
"""
|
|
if listed is None:
|
|
listed = private_files()[0]
|
|
travelling, named_dirs, resident = {}, {}, {}
|
|
for rel in listed:
|
|
slashed = rel.replace('\\', '/')
|
|
for pat, why in pats:
|
|
m = pat.search(slashed)
|
|
if not m:
|
|
continue
|
|
if why in FILENAME_CATEGORIES and \
|
|
_is_placeholder_filename(m.group(0), placeholders):
|
|
continue
|
|
entry = named_dirs.setdefault((why, m.group(0)[:40]), [0, slashed])
|
|
entry[0] += 1
|
|
try:
|
|
with open(os.path.join(ROOT, rel), encoding='utf-8',
|
|
errors='replace') as fh:
|
|
lines = fh.readlines()
|
|
except OSError:
|
|
continue
|
|
kind = _file_kind(rel)
|
|
in_fence = False
|
|
for n, line in enumerate(lines, 1):
|
|
if kind == 'doc' and _FENCE.match(line):
|
|
in_fence = not in_fence
|
|
continue
|
|
for pat, why in pats:
|
|
m = pat.search(line)
|
|
if not m:
|
|
continue
|
|
if why in FILENAME_CATEGORIES and \
|
|
_is_placeholder_filename(m.group(0), placeholders):
|
|
continue
|
|
where = _travels(kind, line, m.start(), why, in_fence)
|
|
if where is None:
|
|
resident[slashed] = resident.get(slashed, 0) + 1
|
|
continue
|
|
if where == 'structural':
|
|
# The pattern already matched the whole string that
|
|
# identifies somebody -- a user path, an address, a model
|
|
# filename. Widening that to the surrounding token only
|
|
# chops it at the first space and prints half of it.
|
|
token = m.group(0)
|
|
else:
|
|
token = _token_at(line, m.start()) \
|
|
.strip('`\'"*,.:;()[]{}') or m.group(0)
|
|
glued = token.lower() != m.group(0).lower()
|
|
# Keyed on the token alone, not on (where, token): one string
|
|
# is one thing to decide about, and the same identifier
|
|
# quoted in a fence in one note, inline in another and in a
|
|
# scratch script in a third is still that one string. The
|
|
# first `where` seen labels it, and structural patterns run
|
|
# before terms, so the more specific label wins.
|
|
entry = travelling.setdefault(
|
|
token[:70],
|
|
[0, '{0}:{1}'.format(slashed, n), why, glued, where])
|
|
entry[0] += 1
|
|
return travelling, named_dirs, resident
|
|
|
|
|
|
def report_private(listed, dropped, travelling, named_dirs, resident,
|
|
show=30) -> None:
|
|
"""Print the three piles: the decidable one first, the expected ones as
|
|
numbers. Every section is capped, because a section that can grow without
|
|
limit is a section that stops being read."""
|
|
skipped = sum(dropped.values())
|
|
print('\nprivate tree -- {0} gitignored file(s) read, {1} not read'
|
|
.format(len(listed), skipped))
|
|
if skipped:
|
|
print(' not read: ' + ', '.join(
|
|
'{0} {1}'.format(v, k) for k, v in sorted(dropped.items()) if v))
|
|
|
|
if travelling:
|
|
# Glued first: the name buried inside a longer identifier is the one
|
|
# a human eye slides over. A bare mention in a code span is still a
|
|
# word somebody can see is a name.
|
|
ordered = sorted(travelling.items(),
|
|
key=lambda kv: (not kv[1][3], -kv[1][0], kv[0]))
|
|
glued_n = sum(1 for _, v in ordered if v[3])
|
|
print('\nTRAVELLING -- {0} distinct string(s) in a shape that gets '
|
|
'copied, {1} of them with the name glued into a longer token. '
|
|
'Legitimate where they are; the question is only whether one '
|
|
'has already been pasted somewhere tracked:'
|
|
.format(len(ordered), glued_n))
|
|
for token, (count, example, why, glued, where) in ordered[:show]:
|
|
print(_safe(' {0}{1}: {2} -- {3} x{4}, e.g. {5}'.format(
|
|
'*' if glued else ' ', where, token, why, count, example)))
|
|
if len(ordered) > show:
|
|
print(' ... and {0} more distinct string(s)'
|
|
.format(len(ordered) - show))
|
|
|
|
if named_dirs:
|
|
ordered = sorted(named_dirs.items(), key=lambda kv: (-kv[1][0], kv[0]))
|
|
print('\nnamed paths -- {0} name(s) in the private tree\'s own file '
|
|
'and folder names. Nothing to fix here, but a tracked file that '
|
|
'quotes one of these paths publishes it:'.format(len(ordered)))
|
|
for (why, text), (count, example) in ordered[:show]:
|
|
print(_safe(' {0}: {1} -- in {2} path(s), e.g. {3}'
|
|
.format(why, text, count, example)))
|
|
if len(ordered) > show:
|
|
print(' ... and {0} more'.format(len(ordered) - show))
|
|
|
|
if resident:
|
|
total = sum(resident.values())
|
|
ordered = sorted(resident.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
print('\nresident -- {0} match(es) in {1} file(s), not listed. A name '
|
|
'in prose, or in a data value, is these files doing their job:'
|
|
.format(total, len(ordered)))
|
|
for rel, count in ordered[:show]:
|
|
print(_safe(' {0:>6} {1}'.format(count, rel)))
|
|
if len(ordered) > show:
|
|
print(' ... and {0} more file(s)'.format(len(ordered) - show))
|
|
|
|
# This mode has no verdict and deliberately returns 0 whatever it finds:
|
|
# see the paragraph above `main`'s --private branch.
|
|
print('\nThis is an inventory, not a gate. Nothing above blocks a commit, '
|
|
'and nothing above is wrong for being where it is -- the question '
|
|
'each row asks is whether that exact string has already been '
|
|
'copied into a tracked file. Check one with:\n'
|
|
' git grep -i -- "<string>"')
|
|
|
|
|
|
# The interpreter is baked in rather than left as bare `python`, and that is
|
|
# not fussiness. On Windows `python` usually resolves to the Microsoft Store
|
|
# alias in WindowsApps: a stub that prints "Python was not found" and exits
|
|
# non-zero. The hook then blocks every commit on this repository with a
|
|
# message about the Store, which looks nothing like a scrub failure and sends
|
|
# you looking in the wrong place entirely.
|
|
#
|
|
# A hook is per-clone and not part of the repository, so pinning the
|
|
# interpreter that installed it is exactly the right scope. The fallback chain
|
|
# is there for a clone whose Python later moves.
|
|
HOOK = """#!/bin/sh
|
|
# Installed by tools/scrub.py --install-hook. Blocks a commit carrying a
|
|
# client name. Bypass with: git commit --no-verify
|
|
for py in "{python}" python3 python; do
|
|
case "$py" in *WindowsApps*|"") continue;; esac
|
|
command -v "$py" >/dev/null 2>&1 || [ -x "$py" ] || continue
|
|
exec "$py" tools/scrub.py --staged
|
|
done
|
|
echo "scrub: no Python found. Reinstall the hook with:" >&2
|
|
echo " python tools/scrub.py --install-hook" >&2
|
|
exit 1
|
|
"""
|
|
|
|
|
|
def install_hook() -> int:
|
|
"""Put the worktree scan in front of every commit on this clone.
|
|
|
|
Hooks are not part of a repository, so this has to be run once per clone
|
|
-- including on the office machine after cloning.
|
|
"""
|
|
hooks = os.path.join(git('rev-parse', '--git-dir').strip() or '.git',
|
|
'hooks')
|
|
if not os.path.isdir(hooks):
|
|
os.makedirs(hooks)
|
|
path = os.path.join(hooks, 'pre-commit')
|
|
with open(path, 'w', encoding='utf-8', newline='\n') as fh:
|
|
fh.write(HOOK.format(python=sys.executable.replace('\\', '/')))
|
|
os.chmod(path, 0o755)
|
|
print('installed pre-commit hook: {0}\n using {1}'
|
|
.format(path, sys.executable))
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument('--worktree', action='store_true',
|
|
help='skip the history scan')
|
|
ap.add_argument('--staged', action='store_true',
|
|
help='what the pre-commit hook runs: hits in STAGED files '
|
|
'block the commit, hits in untracked files are '
|
|
'reported but do not. An untracked file cannot reach '
|
|
'history without being staged first, and blocking on '
|
|
'one stops work that has nothing to do with the '
|
|
'commit -- which is how a gate ends up bypassed with '
|
|
'--no-verify, and then it guards nothing.')
|
|
ap.add_argument('--private', action='store_true',
|
|
help='audit the gitignored tree (QNA/, knowledge/plant/, '
|
|
'knowledge/job/) -- the other repository, where a '
|
|
'name is allowed to be. An inventory, not a gate: '
|
|
'never run by the hook and it always exits 0. It '
|
|
'lists only names sitting in a shape that travels '
|
|
'(a fenced block, a proposed key, a quoted literal, '
|
|
'a path, a name glued into a longer identifier), '
|
|
'aggregated by the distinct string, and counts the '
|
|
'rest without listing them -- a wall of expected '
|
|
'hits is a report nobody reads twice.')
|
|
ap.add_argument('--terms', default=None,
|
|
help="terms file to scan for (default .scrub-terms). CI "
|
|
"passes .scrub-terms.example, the committed generic "
|
|
"sample file, so it never needs the real denylist.")
|
|
ap.add_argument('--install-hook', action='store_true',
|
|
help='install as a pre-commit hook on this clone')
|
|
args = ap.parse_args()
|
|
|
|
if args.install_hook:
|
|
return install_hook()
|
|
|
|
terms = load_terms(args.terms)
|
|
if not terms:
|
|
print('note: no .scrub-terms file, only structural patterns checked.\n'
|
|
' copy .scrub-terms.example and add your client names.',
|
|
file=sys.stderr)
|
|
pats = patterns(terms)
|
|
|
|
placeholders = load_placeholders()
|
|
if not placeholders:
|
|
print("note: could not find CLAUDE.md's placeholder table -- the "
|
|
'model/family-filename patterns will flag the sanctioned '
|
|
'placeholder name too, with nothing exempted.',
|
|
file=sys.stderr)
|
|
|
|
if args.private:
|
|
listed, dropped = private_files()
|
|
travelling, named_paths, resident = scan_private(
|
|
pats, listed, placeholders)
|
|
report_private(listed, dropped, travelling, named_paths, resident)
|
|
# Always 0, deliberately -- see the paragraph in the module docstring.
|
|
# Every string this prints carries a denylist term or a structural
|
|
# pattern, so a copy of one into a tracked file is already a --worktree
|
|
# failure; there is no leak here for an exit code to catch that is not
|
|
# caught one directory out. What is here is the raw material, and a
|
|
# mode that failed on the raw material would fail on every run.
|
|
return 0
|
|
|
|
loose = []
|
|
if args.staged:
|
|
worktree = scan_worktree(pats, staged_files(), placeholders)
|
|
loose = scan_worktree(pats, untracked_files(), placeholders)
|
|
history = []
|
|
else:
|
|
worktree = scan_worktree(pats, placeholders=placeholders)
|
|
history = [] if args.worktree else scan_history(pats, placeholders)
|
|
|
|
label_now = 'staged for this commit' if args.staged else 'working tree'
|
|
for label, hits in ((label_now, worktree), ('git history', history)):
|
|
if hits:
|
|
print('\n{0} -- {1} hit(s):'.format(label, len(hits)))
|
|
for hit in hits:
|
|
print(' ' + hit)
|
|
|
|
if loose:
|
|
# Not fatal, and said every time until it is dealt with. These are one
|
|
# 'git add -A' away from being the blocking kind.
|
|
print('\nWARNING -- {0} hit(s) in untracked files. Not committed, and '
|
|
'not blocking this commit, but one "git add -A" away from '
|
|
'being:'.format(len(loose)))
|
|
for hit in loose:
|
|
print(' ' + hit)
|
|
|
|
if not args.staged:
|
|
# The blind spot, named out loud in every audit run rather than left
|
|
# to be rediscovered. These files are not scanned here and should not
|
|
# be -- a hit in them is not a leak -- but "nothing looks at all" is
|
|
# how a name sits in a note for six weeks and then gets pasted into a
|
|
# module. --private is what looks.
|
|
n_private = len(private_files()[0])
|
|
print('\nnote: {0} readable gitignored file(s) were not scanned here '
|
|
'-- QNA/, knowledge/plant/, knowledge/job/, _scratch/. A name '
|
|
'is allowed in those, so they are not part of this verdict, '
|
|
'but nothing else looks at them either:\n'
|
|
' python tools/scrub.py --private'.format(n_private))
|
|
|
|
if not worktree and not history:
|
|
if args.staged:
|
|
print('clean: {0} staged file(s) checked against {1} term(s) + {2} '
|
|
'structural pattern(s)'
|
|
.format(len(staged_files()), len(terms), len(STRUCTURAL)))
|
|
return 0
|
|
scope = 'working tree' if args.worktree else 'working tree and history'
|
|
print('clean: {0} checked against {1} term(s) + {2} structural pattern(s)'
|
|
.format(scope, len(terms), len(STRUCTURAL)))
|
|
return 0
|
|
|
|
if history:
|
|
print('\nA hit in history is not fixed by editing the file. If nothing '
|
|
'has been pushed yet, rewrite the history; if it has, the name '
|
|
'is public and rotating the repo is the only real fix.')
|
|
return 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|