will
8e6ee31596
ci / test (push) Successful in 6m18s
History-free snapshot of the private development repository at cbe45c5 (2026-09-09). Code, docs and public catalogue only; no plant or job knowledge layers, no client or job names (tools/scrub.py clean). The intro deck is left out of this snapshot because its slide images could not be checked for job drawings.
379 lines
15 KiB
Python
379 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""Strict-parse every knowledge layer on this machine. Exit 1 if one will not.
|
|
|
|
python tools/knowledgelint.py # every standard layer here
|
|
python tools/knowledgelint.py a.yaml b.yaml # just these
|
|
python tools/knowledgelint.py knowledge/ # everything under a folder
|
|
python tools/knowledgelint.py --list # what it would check
|
|
python tools/knowledgelint.py --json # machine readable
|
|
|
|
**AFTER EDITING ANY KNOWLEDGE LAYER, RUN THIS.** A landing agent that writes
|
|
a fact into `knowledge/plant/<plant>/knowledge.yaml`, a job file, the
|
|
catalogue or a project config is not done until this passes. It takes under a
|
|
second and it is the only thing between a mistyped file and a whole layer
|
|
disappearing inside Revit.
|
|
|
|
WHY IT IS A GATE, AND WHY `scrub.py --private` IS NOT
|
|
-----------------------------------------------------
|
|
`--private` is an inventory: what it prints is raw material, and a check that
|
|
fails on every run gets wrapped in `|| true`. This one is the opposite. An
|
|
unparseable knowledge layer is ALWAYS wrong -- there is no state of the world
|
|
in which a file the tool cannot read is acceptable -- so a failure here blocks,
|
|
and exit 1 means exactly that.
|
|
|
|
WHAT IT IS FOR (measured 2026-09-01)
|
|
------------------------------------
|
|
pyRevit's reader is YamlDotNet, and it is STRICT: a mapping key stated twice
|
|
raises `Duplicate key` and the read returns nothing. `knowledge._merge_file`
|
|
wrapped that read in a bare `except Exception: return merged`, so one nested
|
|
duplicate in one plant file silently discarded the ENTIRE 255KB plant layer.
|
|
Every parameter role mapped to None, the sweep enumerated 0 assemblies, and
|
|
every report rendered normally -- `sources()` and `config_problems()` both
|
|
said healthy. A layer that fails to load looks exactly like a layer that
|
|
agrees with the defaults.
|
|
|
|
`_merge_file` now records `_<layer>_unreadable` and `config_problems()` refuses
|
|
on it, so Revit will say so. But that is the late warning, one button press
|
|
before a wrong report; this is the early one, on the machine where the edit was
|
|
made, before Revit is even open.
|
|
|
|
Host CPython 3.x, PyYAML. It reads with a unique-key SafeLoader -- the same
|
|
strictness YamlDotNet has and the same one `tests/pyrevit_yaml_fake.py`
|
|
doubles -- and NOT with `vibe.yamlkit`, which is a hand-rolled subset that
|
|
takes the last of two duplicate keys and says nothing.
|
|
|
|
A NOTE ON WHAT IT PRINTS
|
|
------------------------
|
|
It names files, including the gitignored plant and job layers, because a lint
|
|
tool that will not tell you which file failed is not a lint tool. Those paths
|
|
carry job and client names. The output is for your terminal: do not paste it
|
|
into a tracked file, a commit message or a doc.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import io
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
try:
|
|
import yaml
|
|
except ImportError: # pragma: no cover
|
|
sys.stderr.write('PyYAML is required: pip install pyyaml\n')
|
|
raise SystemExit(2)
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
CATALOGUE = os.path.join(ROOT, 'knowledge', 'catalogue.yaml')
|
|
CATALOGUE_TEMPLATE = os.path.join(ROOT, 'knowledge', 'catalogue.yaml.template')
|
|
|
|
# Mirrors `vibe.plantpath` / `vibe.designspec`, which cannot be imported here:
|
|
# `vibe/__init__.py` pulls in the Revit API.
|
|
PLANT_ENV = 'WILLTOOLS_KNOWLEDGE'
|
|
PLANT_FOLDER = 'WillTools'
|
|
JOBS_ENV = 'WILLTOOLS_JOBS'
|
|
LAYER_FILE = 'knowledge.yaml'
|
|
CONFIG_POINTER = 'active-config.txt'
|
|
|
|
SKIP_DIRS = {'__pycache__', '.git', 'dist', '_archive', '_scratch',
|
|
'.pytest_cache', 'node_modules', 'library'}
|
|
YAML_SUFFIXES = ('.yaml', '.yml')
|
|
|
|
|
|
class DuplicateKeyError(yaml.YAMLError):
|
|
"""A mapping key stated twice -- what YamlDotNet refuses and PyYAML allows.
|
|
|
|
Named as its own class so a caller can tell the failure this tool exists
|
|
for from an ordinary syntax error, and worded to lead with the same two
|
|
words pyRevit's reader uses (`Duplicate key`) so the string a person sees
|
|
here is the string they will see in Self Check.
|
|
"""
|
|
|
|
|
|
def _unique_key_loader():
|
|
"""SafeLoader that raises on a duplicate mapping key at ANY depth.
|
|
|
|
The standard recipe: `construct_mapping` runs once per mapping node, so
|
|
nesting comes free, and `key_node.start_mark` points at the SECOND
|
|
statement of the key -- the line to delete.
|
|
"""
|
|
class UniqueKeyLoader(yaml.SafeLoader):
|
|
pass
|
|
|
|
def construct_mapping(loader, node, deep=False):
|
|
seen = set()
|
|
for key_node, _value in node.value:
|
|
key = loader.construct_object(key_node, deep=deep)
|
|
try:
|
|
duplicate = key in seen
|
|
except TypeError: # unhashable key: not this question
|
|
continue
|
|
if duplicate:
|
|
mark = key_node.start_mark
|
|
raise DuplicateKeyError(
|
|
'Duplicate key: {0!r}, stated again at line {1}, column '
|
|
'{2}. A mapping may state a key only once; pyRevit\'s '
|
|
'reader refuses the WHOLE FILE over it, so the layer is '
|
|
'not merged at all.'.format(
|
|
key, mark.line + 1, mark.column + 1))
|
|
seen.add(key)
|
|
return yaml.SafeLoader.construct_mapping(loader, node, deep=deep)
|
|
|
|
UniqueKeyLoader.add_constructor(
|
|
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping)
|
|
return UniqueKeyLoader
|
|
|
|
|
|
def check_file(path):
|
|
"""(ok, message) for one file. `ok` False means the tool cannot read it.
|
|
|
|
The message carries line and column wherever the parser gives them --
|
|
a duplicate key in a 4,000-line plant file is otherwise a search.
|
|
"""
|
|
try:
|
|
handle = io.open(path, 'r', encoding='utf-8')
|
|
except IOError as ex:
|
|
return False, 'cannot open: {0}'.format(ex)
|
|
try:
|
|
text = handle.read()
|
|
except (IOError, UnicodeDecodeError) as ex:
|
|
return False, 'cannot read: {0}'.format(ex)
|
|
finally:
|
|
handle.close()
|
|
|
|
try:
|
|
yaml.load(text, Loader=_unique_key_loader())
|
|
except DuplicateKeyError as ex:
|
|
return False, str(ex)
|
|
except yaml.YAMLError as ex:
|
|
return False, _yaml_message(ex)
|
|
except Exception as ex: # pragma: no cover
|
|
return False, '{0}: {1}'.format(type(ex).__name__, ex)
|
|
return True, 'ok'
|
|
|
|
|
|
def _yaml_message(ex):
|
|
"""A PyYAML error as one line, with the line/column it knows about."""
|
|
problem = (getattr(ex, 'problem', None) or str(ex)).strip()
|
|
mark = getattr(ex, 'problem_mark', None)
|
|
if mark is None:
|
|
return problem
|
|
context = (getattr(ex, 'context', None) or '').strip()
|
|
where = 'line {0}, column {1}'.format(mark.line + 1, mark.column + 1)
|
|
if context:
|
|
return '{0} ({1}) at {2}'.format(problem, context, where)
|
|
return '{0} at {1}'.format(problem, where)
|
|
|
|
|
|
# ------------------------------------------------------------------ targets
|
|
|
|
def plant_root():
|
|
"""The folder the plant layers live in. Mirrors `plantpath.plant_root`.
|
|
|
|
Set to a folder, that is the root; set to a file, its directory is (the
|
|
machine-wide file lives IN the root); unset, %APPDATA%/WillTools.
|
|
"""
|
|
override = os.environ.get(PLANT_ENV)
|
|
if not override:
|
|
base = os.environ.get('APPDATA') or os.path.expanduser('~')
|
|
return os.path.join(base, PLANT_FOLDER)
|
|
if os.path.isdir(override):
|
|
return override
|
|
return os.path.dirname(override) or override
|
|
|
|
|
|
def default_targets():
|
|
"""Every standard layer location that EXISTS on this machine.
|
|
|
|
The four layers `knowledge.load()` merges, resolved the way it resolves
|
|
them minus the document it does not have:
|
|
|
|
* `knowledge/catalogue.yaml`, in this repo;
|
|
* the plant root -- `<root>/knowledge.yaml` and every
|
|
`<root>/<plant>/knowledge.yaml`;
|
|
* every `<jobs root>/<job>/knowledge.yaml`, plus the project config each
|
|
job's `active-config.txt` points at;
|
|
* and, whatever the environment says, everything under this checkout's
|
|
own `knowledge/plant/` and `knowledge/job/`, because that is where
|
|
these files are edited here and an agent whose environment is not set
|
|
up must still be checking the files it just wrote.
|
|
|
|
Deduplicated by real path and returned in that order. A `.bak-*` snapshot
|
|
is skipped: nothing reads one, and yesterday's file failing would make
|
|
this a check people learn to ignore.
|
|
"""
|
|
out = []
|
|
_add(out, CATALOGUE)
|
|
|
|
root = plant_root()
|
|
_add(out, os.path.join(root, LAYER_FILE))
|
|
for name in _entries(root):
|
|
_add(out, os.path.join(root, name, LAYER_FILE))
|
|
|
|
jobs = os.environ.get(JOBS_ENV)
|
|
if jobs:
|
|
for name in _entries(jobs):
|
|
_add(out, os.path.join(jobs, name, LAYER_FILE))
|
|
_add(out, _config_pointed_at(os.path.join(jobs, name)))
|
|
|
|
for folder in ('plant', 'job'):
|
|
for path in walk_yaml(os.path.join(ROOT, 'knowledge', folder)):
|
|
_add(out, path)
|
|
return out
|
|
|
|
|
|
def _add(out, path):
|
|
"""Append an existing file, once, by real path."""
|
|
if not path or not os.path.isfile(path):
|
|
return out
|
|
real = os.path.realpath(path)
|
|
if real not in [os.path.realpath(p) for p in out]:
|
|
out.append(path)
|
|
return out
|
|
|
|
|
|
def _entries(folder):
|
|
try:
|
|
return sorted(os.listdir(folder))
|
|
except OSError:
|
|
return []
|
|
|
|
|
|
def _config_pointed_at(job_dir):
|
|
"""The project config a job's one-line pointer names, if it is there.
|
|
|
|
A pointer to a file that is GONE is `knowledge._config_missing`, which the
|
|
buttons already refuse on, and it is not a parse failure -- the file may
|
|
simply live on a share this machine is not on today. So it is not this
|
|
tool's verdict: it returns None and the run stays about parseability.
|
|
"""
|
|
pointer = os.path.join(job_dir, CONFIG_POINTER)
|
|
if not os.path.isfile(pointer):
|
|
return None
|
|
try:
|
|
handle = io.open(pointer, 'r', encoding='utf-8')
|
|
except IOError:
|
|
return None
|
|
try:
|
|
target = handle.read().strip()
|
|
finally:
|
|
handle.close()
|
|
return target or None
|
|
|
|
|
|
def walk_yaml(folder):
|
|
"""Every YAML file under a folder, sorted, without the `.bak-` snapshots."""
|
|
out = []
|
|
if not os.path.isdir(folder):
|
|
return out
|
|
for dirpath, dirnames, filenames in os.walk(folder):
|
|
dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS)
|
|
for name in sorted(filenames):
|
|
if name.endswith(YAML_SUFFIXES) and '.bak-' not in name:
|
|
out.append(os.path.join(dirpath, name))
|
|
return out
|
|
|
|
|
|
def resolve(paths):
|
|
"""Command-line arguments as a list of files. A folder means everything in it."""
|
|
out = []
|
|
for path in paths:
|
|
if os.path.isdir(path):
|
|
for found in walk_yaml(path):
|
|
_add(out, found)
|
|
elif os.path.isfile(path):
|
|
_add(out, path)
|
|
else:
|
|
out.append(path) # kept, and reported as unreadable
|
|
return out
|
|
|
|
|
|
# --------------------------------------------------------------------- main
|
|
|
|
def run(paths):
|
|
"""[(path, ok, message)] for everything asked for, in order."""
|
|
return [(path,) + check_file(path) for path in paths]
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(
|
|
description='Strict-parse the knowledge layers. Exit 1 if one fails.')
|
|
parser.add_argument('paths', nargs='*',
|
|
help='files or folders; default: every standard '
|
|
'layer location that exists on this machine')
|
|
parser.add_argument('--list', action='store_true',
|
|
help='print what would be checked and stop')
|
|
parser.add_argument('--json', action='store_true', dest='as_json',
|
|
help='machine-readable results')
|
|
args = parser.parse_args(argv)
|
|
|
|
targets = resolve(args.paths) if args.paths else default_targets()
|
|
|
|
if args.list:
|
|
for path in targets:
|
|
print(path)
|
|
return 0
|
|
|
|
if not targets:
|
|
# Not a pass, ordinarily: nothing was checked, and saying "ok" to
|
|
# that is the exact collapse this repository keeps paying for on the
|
|
# owner's own machines, where knowledge/catalogue.yaml is always
|
|
# tracked and this branch should never be reachable.
|
|
#
|
|
# ONE EXCEPTION, and it is checked by absence of the file this
|
|
# checkout SHOULD have plus presence of the file that stands in for
|
|
# it on a stripped install (see tools/make_dist.py): a colleague
|
|
# trial distribution genuinely ships no knowledge layer at all, on
|
|
# the owner's own instruction, and "nothing to check" is the honest,
|
|
# expected answer there -- SKIP-shaped, not FAIL-shaped, same
|
|
# discipline tools/setup_machine.py --check applies to the same
|
|
# state. This is the only branch where exit 0 with nothing checked
|
|
# is not a lie.
|
|
if (not args.paths and os.path.isfile(CATALOGUE_TEMPLATE)
|
|
and not os.path.isfile(CATALOGUE)):
|
|
print('no knowledge layers configured -- checks that need them '
|
|
'stand down. This looks like a knowledge-less trial '
|
|
'install ({0} is present in place of the real catalogue): '
|
|
'expected here, not a failure. Ask the owner for '
|
|
'knowledge/catalogue.yaml plus a plant/job layer to check '
|
|
'them for real.'.format(CATALOGUE_TEMPLATE))
|
|
return 0
|
|
print('no knowledge layer found to check -- neither {0} nor {1} is '
|
|
'set and this checkout has no plant or job files. Name the '
|
|
'files to check, or set the variables.'.format(PLANT_ENV,
|
|
JOBS_ENV))
|
|
return 1
|
|
|
|
results = run(targets)
|
|
failed = [r for r in results if not r[1]]
|
|
|
|
if args.as_json:
|
|
print(json.dumps(
|
|
{'checked': len(results), 'failed': len(failed),
|
|
'results': [{'path': p, 'ok': ok, 'message': msg}
|
|
for p, ok, msg in results]},
|
|
indent=2))
|
|
return 1 if failed else 0
|
|
|
|
for path, ok, message in results:
|
|
print('{0} {1}'.format('ok ' if ok else 'FAILED', path))
|
|
if not ok:
|
|
print(' {0}'.format(message))
|
|
|
|
print('')
|
|
if failed:
|
|
print('{0} of {1} layer file(s) WILL NOT LOAD.'.format(len(failed),
|
|
len(results)))
|
|
print('pyRevit refuses the whole file, so the whole layer is dropped '
|
|
'and every value in it silently falls back -- which looks '
|
|
'exactly like a layer that agrees with the defaults. Fix the '
|
|
'file; nothing will guess at what it said.')
|
|
return 1
|
|
print('{0} layer file(s) parse.'.format(len(results)))
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|