will
52d94303bc
ci / test (push) Has been cancelled
Public demo snapshot of an in-house pyRevit extension for precast concrete shop-ticket QC (37 checks / 46 checklist rows) and a standalone panel handling solver (PCI Chapter 8). - No client, job, plant or product data: knowledge layers ship as structural templates only. - Ticket QC's model-read escalation targets a LAN model server endpoint (configurable), not a cloud provider. - Built from the private development repo via tools/make_dist.py.
769 lines
33 KiB
Python
769 lines
33 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""The whole configuration of one project, written to one file.
|
|
|
|
Knowledge arrives in five layers -- built-in defaults, the catalogue, the
|
|
plant, the job, and the project config file -- and every one of them but the
|
|
defaults lives somewhere nobody browses to: the tool's own install, APPDATA,
|
|
a folder beside the model, and wherever the engineer chose to put the config
|
|
file. That is the right way to store it and the wrong way to hand it to
|
|
somebody. A reviewer asking "what was this checked against?" should get one
|
|
file, not a tour of three machines.
|
|
|
|
So this flattens the layers into a single document that says, for every
|
|
setting, what the value is and which layer it came from. `layer_values()` is
|
|
where the five are enumerated, and it is also where the one subtlety lives:
|
|
the project config layer is left out of the provenance by default, for the
|
|
reason set out in that function's own docstring. The knowledge section
|
|
is written in the same shape a knowledge.yaml has, which means the file is not
|
|
only a record: save it as the job's knowledge file and it reproduces exactly
|
|
the configuration it was taken from.
|
|
|
|
Written for IronPython 2.7: no f-strings, no pathlib, no OrderedDict literals.
|
|
"""
|
|
# pylint: disable=import-error,invalid-name,broad-except
|
|
import datetime
|
|
import os
|
|
import re
|
|
|
|
from vibe import knowledge
|
|
from vibe.compat import string_types, text_type
|
|
|
|
# The order sections come out in -- the order somebody setting a project up
|
|
# would work through them, which is not alphabetical. Anything not listed
|
|
# follows, sorted, so a new knowledge key never silently vanishes from a dump
|
|
# written by an older build.
|
|
SECTION_ORDER = (
|
|
'nomenclature', 'parameters', 'families', 'orientation', 'rails',
|
|
'legend', 'sheet', 'concrete', 'lifters', 'handling', 'interference',
|
|
'catalogue',
|
|
)
|
|
|
|
SECTION_NOTES = {
|
|
'nomenclature': 'How a design number is written, and what reduces to what',
|
|
'orientation': 'Which end of a piece is the top once it is erected',
|
|
'rails': 'The casting bed: which rail is fixed, how wide the bed is',
|
|
'parameters': "This office's Revit parameter name for each role the "
|
|
'tool reads',
|
|
'families': 'Which family names count as strand, mesh, or a lifting insert',
|
|
'lifters': 'Part number -> catalogue designation, and the checks on top',
|
|
'handling': ('Lifting and handling: stages, insert roles, where reports '
|
|
'go. rig_mode and dunnage are decided in the decisions '
|
|
'section at the top; a stage with no rig_mode here inherits '
|
|
'the default.'),
|
|
'interference': 'The clash check between members of one assembly',
|
|
'catalogue': 'Published hardware ratings. Not job data -- ships with the '
|
|
'tool',
|
|
'legend': 'The strand-pattern legend a ticket has to carry',
|
|
'sheet': 'What is compared against the printed ticket',
|
|
'concrete': 'Release strength rules',
|
|
}
|
|
|
|
|
|
def sections(know):
|
|
"""Knowledge section names, in reading order, nothing dropped.
|
|
|
|
Except the underscore keys. `_config_missing` and `_config_problems` are
|
|
notes load() leaves for config_problems ABOUT this run -- not settings --
|
|
and writing them into the file would make a transient complaint look like
|
|
a permanent setting, and hand it back to the next merge as one.
|
|
"""
|
|
listed = [key for key in SECTION_ORDER if key in know]
|
|
rest = sorted(key for key in know
|
|
if key not in SECTION_ORDER and not str(key).startswith('_'))
|
|
return listed + rest
|
|
|
|
|
|
# ------------------------------------------------------------ a YAML writer
|
|
|
|
_PLAIN = re.compile(r'^[A-Za-z0-9_][A-Za-z0-9_ ./+-]*$')
|
|
_RESERVED = ('true', 'false', 'null', 'yes', 'no', 'on', 'off', '~')
|
|
|
|
|
|
def _scalar(value):
|
|
if value is None:
|
|
return 'null'
|
|
if value is True:
|
|
return 'true'
|
|
if value is False:
|
|
return 'false'
|
|
if isinstance(value, float):
|
|
text = repr(value)
|
|
return text[:-2] if text.endswith('.0') else text
|
|
if not isinstance(value, string_types):
|
|
return str(value)
|
|
|
|
text = text_type(value)
|
|
if not text:
|
|
return "''"
|
|
if text.lower() in _RESERVED or not _PLAIN.match(text) \
|
|
or text != text.strip() or text.endswith(':'):
|
|
return "'" + text.replace("'", "''") + "'"
|
|
# A bare number-looking string has to be quoted or it reads back as a
|
|
# number, and a part number like 4.10 would come back as 4.1.
|
|
if re.match(r'^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$', text):
|
|
return "'" + text + "'"
|
|
return text
|
|
|
|
|
|
def _is_flat(node):
|
|
"""A list of plain scalars, short enough to sit on one line."""
|
|
if not isinstance(node, list) or not node:
|
|
return False
|
|
if any(isinstance(v, (dict, list)) for v in node):
|
|
return False
|
|
return sum(len(_scalar(v)) + 2 for v in node) <= 66
|
|
|
|
|
|
def emit(node, indent=0):
|
|
"""A dict/list/scalar tree as YAML lines. Deterministic key order."""
|
|
pad = ' ' * indent
|
|
out = []
|
|
|
|
if isinstance(node, dict):
|
|
if not node:
|
|
return [pad + '{}']
|
|
for key in sorted(node):
|
|
value = node[key]
|
|
label = pad + _scalar(key) + ':'
|
|
if isinstance(value, dict) and value:
|
|
out.append(label)
|
|
out.extend(emit(value, indent + 1))
|
|
elif isinstance(value, list) and value:
|
|
if _is_flat(value):
|
|
out.append('{0} [{1}]'.format(
|
|
label, ', '.join(_scalar(v) for v in value)))
|
|
else:
|
|
out.append(label)
|
|
out.extend(emit(value, indent + 1))
|
|
elif isinstance(value, (dict, list)):
|
|
out.append('{0} {1}'.format(label,
|
|
'{}' if isinstance(value, dict)
|
|
else '[]'))
|
|
else:
|
|
out.append('{0} {1}'.format(label, _scalar(value)))
|
|
return out
|
|
|
|
if isinstance(node, list):
|
|
for item in node:
|
|
if isinstance(item, dict) and item:
|
|
lines = emit(item, indent + 1)
|
|
# The first key rides on the dash, the rest line up under it.
|
|
out.append(pad + '- ' + lines[0].strip())
|
|
out.extend(lines[1:])
|
|
elif isinstance(item, list) and item:
|
|
lines = emit(item, indent + 1)
|
|
out.append(pad + '-')
|
|
out.extend(lines)
|
|
else:
|
|
out.append(pad + '- ' + _scalar(item))
|
|
return out
|
|
|
|
return [pad + _scalar(node)]
|
|
|
|
|
|
# --------------------------------------------------------------- provenance
|
|
|
|
def _layer_rows(doc, include_config=False):
|
|
"""[(label, path, exists, data, unreadable)] -- what layer_values() and
|
|
layer_problems() are both built out of.
|
|
|
|
`unreadable` is `None` for an ordinary layer (absent, or present and read
|
|
clean) and the parser's message (a non-empty string) for a layer whose
|
|
file IS there and would not parse.
|
|
|
|
This used to be `layer_values()` itself, calling
|
|
`knowledge._merge_file(holder, path)` with NO `layer` argument -- which
|
|
means `_note_layer_unreadable` returned immediately (`if not layer: ...`)
|
|
and a file that exists and will not parse came back as `data = {}`,
|
|
indistinguishable from a file that genuinely sets nothing. The five-layer
|
|
merge in `knowledge.load()` was never fooled by this -- it always passes
|
|
the token -- so a real run's `_plant_unreadable` marker was sitting right
|
|
there in `know`; only THIS dump's own, separate re-read of the files
|
|
(needed to say which layer a value came from) was blind to it. Threading
|
|
the token through here, and popping the marker back out of `data` before
|
|
handing it to a caller, fixes that without changing what a marker-free
|
|
`data` dict means to `origins()` / `overridden()` / every existing caller
|
|
of `layer_values()`.
|
|
"""
|
|
from vibe import designspec
|
|
layers = [('defaults', None, True,
|
|
knowledge._deep_copy(knowledge.DEFAULTS), None)]
|
|
# The plant layer is per plant, and which plant is a key in the JOB layer
|
|
# -- so it has to be resolved through the same pre-pass load() uses, or
|
|
# this dump attributes settings to a file the run never read.
|
|
# The token doubles as the label for these three -- 'catalogue', 'plant'
|
|
# and 'job' are both what `_FILE_LAYERS` calls the layer and what a
|
|
# human-readable line calls it.
|
|
for label, path in (('catalogue', knowledge.catalogue_path()),
|
|
('plant',
|
|
designspec.plant_knowledge_path(
|
|
knowledge.plant_id(doc))),
|
|
('job', designspec.knowledge_path(doc))):
|
|
layer = label
|
|
data = {}
|
|
unreadable = None
|
|
exists = bool(path and os.path.isfile(path))
|
|
if exists:
|
|
holder = {}
|
|
knowledge._merge_file(holder, path, layer)
|
|
marker = holder.pop(knowledge._unreadable_key(layer), None)
|
|
if marker:
|
|
unreadable = marker[1]
|
|
data = holder
|
|
layers.append((label, path, exists, data, unreadable))
|
|
|
|
if include_config:
|
|
path = designspec.active_config_path(doc)
|
|
if path:
|
|
data = {}
|
|
unreadable = None
|
|
exists = os.path.isfile(path)
|
|
if exists:
|
|
# Through _merge_config_file, not _merge_file: the file's
|
|
# `decisions:` block is mapped back onto the keys really read,
|
|
# and it beats the tree below it inside the same file. A
|
|
# provenance view that read only the tree would name the wrong
|
|
# value for the eight settings that have both spellings.
|
|
# _merge_config_file already threads 'config' through to
|
|
# _merge_file on its own (`notes=merged`), so the marker
|
|
# lands on `holder` here the same way it does for the other
|
|
# three layers -- just popped back out below instead.
|
|
holder = {}
|
|
knowledge._merge_config_file(holder, path)
|
|
marker = holder.pop(knowledge._unreadable_key('config'), None)
|
|
if marker:
|
|
unreadable = marker[1]
|
|
data = holder
|
|
layers.append(('config', path, exists, data, unreadable))
|
|
return layers
|
|
|
|
|
|
def layer_values(doc, include_config=False):
|
|
"""[(label, path, exists, data)] for every layer, in the order they merge.
|
|
|
|
Read separately from load() so the dump can say which layer a value came
|
|
from. Saying only what a setting ended up as leaves the reader unable to
|
|
tell a deliberate plant rule from a default nobody has looked at.
|
|
|
|
The project config layer is deliberately NOT included by default. This is
|
|
what writes it: folding the previous version of the file into the
|
|
provenance would stamp every setting as coming from the file about to be
|
|
overwritten, and the answer would be true and useless. What a reader wants
|
|
is which of the underlying layers each setting really came from.
|
|
|
|
`include_config=True` adds it, for the one caller that needs the opposite
|
|
answer: an editor showing what is in force has to show the layer that is
|
|
WINNING, and while a config file is live that is the config file. Leaving
|
|
it out there would show somebody a plant value that nothing is reading.
|
|
The flag is additive rather than a change of default so the dump's own
|
|
provenance keeps meaning what it has always meant.
|
|
|
|
`data` never carries an `_<layer>_unreadable` marker -- `_layer_rows()`
|
|
pops it back out before this wraps the row. A caller that wants to know
|
|
whether a layer was unreadable wants `layer_problems()`, not a key it has
|
|
to know to go looking for inside `data`.
|
|
"""
|
|
return [(label, path, exists, data)
|
|
for label, path, exists, data, _unreadable
|
|
in _layer_rows(doc, include_config)]
|
|
|
|
|
|
def layer_problems(doc, include_config=False):
|
|
"""[(label, path, message)] for every layer whose file is on this machine
|
|
and would not parse -- in merge order, empty in the ordinary case.
|
|
|
|
This is the dump's own version of `knowledge.unreadable_layers()`: same
|
|
question, asked against the fresh per-file read `layer_values()` does
|
|
rather than against an already-merged `know` tree, because the dump
|
|
reads the layers itself (see `layer_values()`'s docstring) and has to be
|
|
right on its own -- a caller here may not have a merged tree with the
|
|
markers already on it.
|
|
"""
|
|
return [(label, path, unreadable)
|
|
for label, path, _exists, _data, unreadable
|
|
in _layer_rows(doc, include_config)
|
|
if unreadable]
|
|
|
|
|
|
def _paths(node, prefix=''):
|
|
"""Every leaf in a tree as a dotted path. Lists count as one leaf."""
|
|
found = []
|
|
if isinstance(node, dict):
|
|
for key in node:
|
|
found.extend(_paths(node[key],
|
|
prefix + '.' + str(key) if prefix else str(key)))
|
|
elif prefix:
|
|
found.append(prefix)
|
|
return found
|
|
|
|
|
|
def origins(doc):
|
|
"""{dotted key: the label of the LAST layer that set it}."""
|
|
out = {}
|
|
for label, _path, _exists, data in layer_values(doc):
|
|
for path in _paths(data):
|
|
out[path] = label
|
|
return out
|
|
|
|
|
|
def overridden(doc):
|
|
"""[(key, [(layer, value), ...])] for settings a later layer CHANGES.
|
|
|
|
The interesting rows: where a plant rule beats a default, or a job beats
|
|
its plant. A layer that restates a value it agrees with is not an
|
|
override, and listing it would bury the handful of rows that matter under
|
|
a hundred that do not.
|
|
"""
|
|
seen = {}
|
|
for label, _path, _exists, data in layer_values(doc):
|
|
for path in _paths(data):
|
|
seen.setdefault(path, []).append(
|
|
(label, knowledge.get(data, path)))
|
|
rows = []
|
|
for path in sorted(seen):
|
|
chain = seen[path]
|
|
values = [value for _label, value in chain]
|
|
if len(chain) > 1 and any(v != values[0] for v in values[1:]):
|
|
rows.append((path, chain))
|
|
return rows
|
|
|
|
|
|
# ------------------------------------------------------------ the document
|
|
|
|
def project_info(doc, know):
|
|
"""What piece of work this configuration belongs to."""
|
|
from vibe import designspec, ticketqc
|
|
|
|
info = {
|
|
'model': doc.Title,
|
|
'model_path': doc.PathName or '(unsaved)',
|
|
'qc_data_dir': designspec.root_dir(doc),
|
|
'written': datetime.datetime.now().strftime('%Y-%m-%d %H:%M'),
|
|
}
|
|
try:
|
|
grouped = ticketqc.assemblies_by_design(doc, know)
|
|
specs = designspec.list_specs(doc) if doc.PathName else {}
|
|
designs = []
|
|
for key in sorted(grouped):
|
|
designs.append({
|
|
'design': key,
|
|
'pieces': len(grouped[key]),
|
|
'extracted_spec': key in specs,
|
|
})
|
|
info['designs'] = designs
|
|
info['piece_count'] = sum(len(v) for v in grouped.values())
|
|
except Exception as exc:
|
|
info['designs'] = []
|
|
info['designs_error'] = str(exc)
|
|
return info
|
|
|
|
|
|
DECISION_BANNER = u'''# =========================================================================
|
|
# YOUR DECISIONS
|
|
#
|
|
# Everything in this section is yours. None of it can be worked out from the
|
|
# model, and the tool will not guess at any of it -- it uses what is written
|
|
# here. Edit a value and the next run uses it.
|
|
#
|
|
# Everything after this section is either a record of how this file was built
|
|
# or the merged knowledge it was built from. Worth reading, rarely worth
|
|
# editing: anything named down there and also named up here is overruled by
|
|
# the copy up here.
|
|
# ========================================================================='''
|
|
|
|
|
|
# The comment for each decision, written directly above the value it belongs
|
|
# to rather than gathered into one block at the top of the section. A note
|
|
# four lines away from the line it describes is a note that gets edited past.
|
|
DECISION_NOTES = (
|
|
('report_dir', u'''# Where Panel Lift Check writes its HTML. Used exactly as written, so an
|
|
# absolute path anywhere works -- a job folder on a share, say. Takes effect
|
|
# on the next run.'''),
|
|
|
|
('open_report_when_done', u'''# Open each report as it is finished. The gap between a report being written
|
|
# and being read is one double-click that is easy not to take. Turn it off
|
|
# when working through a lot of pieces in a row, or forty browser tabs is the
|
|
# result.'''),
|
|
|
|
('prestress_loss_pct', u'''# Everything lost between the jack and the stage being checked, as one
|
|
# percentage. It is NOT calculated: a loss model needs an age, a humidity and
|
|
# a curing schedule this tool does not have, and a plausible-looking number
|
|
# from guessed inputs is worse than a figure you can argue with.
|
|
#
|
|
# 8 is conservative. A commercial check of one real piece computed 1.6 at
|
|
# release. LOWER IT AND EVERY PIECE GETS MORE CREDIT.'''),
|
|
|
|
('transfer_length_in', u'''# Where a cut strand has handed its force back to the concrete. Blank uses
|
|
# ACI 318's f_se * db / 3, worked out from the pull stated on the design
|
|
# rather than assumed.'''),
|
|
|
|
('insert_position', u'''# How an insert's position is measured, per run mode.
|
|
#
|
|
# VALID VALUES -- write exactly one of these:
|
|
# fast the element's cached bounding box centre. No geometry solve.
|
|
# precise the solved solid centroid.
|
|
#
|
|
# They agree on a symmetric embed and disagree by a few inches on an L-shaped
|
|
# one -- and a face lifter's position IS the moment arm it hangs on. On-edge
|
|
# anchors are snapped to the edge either way. 'precise' is roughly an order of
|
|
# magnitude slower.
|
|
#
|
|
# The two modes are set apart on purpose. A batch runs unattended, so the time
|
|
# costs nobody anything and it should be right. One piece at a time is
|
|
# somebody waiting at the screen -- and a piece close enough for a few inches
|
|
# to decide it is a piece to put through Eriksson, not to squeeze a sharper
|
|
# number out of here.
|
|
#
|
|
# one_piece Panel Lift Check on a single selected assembly
|
|
# batch Panel Lift Check on several, the sweep, and the run Ticket QC
|
|
# starts for itself'''),
|
|
|
|
('rig_mode', u'''# How the piece hangs. This changes the mechanics, not just a coefficient.
|
|
#
|
|
# VALID VALUES -- write exactly one of these:
|
|
# rolling equalising blocks / roller sheaves. Every line in a cable group
|
|
# carries the same force; two groups, statically determinate.
|
|
# static fixed-length slings / static lift. A continuous beam on rigid
|
|
# supports; the reactions follow stiffness, so a blockout beside a
|
|
# support really does change them.
|
|
#
|
|
# NOT VALID: static_lift, static_lifting, fixed, 1, yes. A misspelled value is
|
|
# REFUSED, not silently inherited -- a typo that looked like "inherit" is
|
|
# exactly the kind of quiet wrongness this section exists to prevent.
|
|
#
|
|
# "default" applies to every stage. A stage named below overrides it for that
|
|
# stage only; put null / leave it blank there to inherit the default.
|
|
#
|
|
# Example -- strip with static slings, erect with equalising blocks:
|
|
# default: rolling
|
|
# stripping: static
|
|
# transportation: null
|
|
# erection: rolling
|
|
#
|
|
# Worth settling before anything else is tuned. A commercial check of one real
|
|
# piece stripped on fixed slings and erected on an equalising block, and its
|
|
# stripping moment came out 14.1 k-ft where one rolling mode everywhere gives
|
|
# 23.8. Which is right depends on how YOUR yard rigs -- nothing here guesses
|
|
# it.'''),
|
|
|
|
('group_tolerance_ft', u'''# How far apart ALONG THE PIECE two inserts may sit and still count as one
|
|
# pick row.
|
|
#
|
|
# The beam solve hangs each ROW from one rigid support line -- but the job
|
|
# names individual inserts and nothing names rows, so this tolerance is what
|
|
# decides who belongs together. Top view, x running along the piece's length:
|
|
#
|
|
# x=14.92 x=15.92
|
|
# | |
|
|
# edge o--+ | the edge anchor and face insert B are flush;
|
|
# B o--+ | C belongs to the SAME row but sits 1.00 ft
|
|
# | C o---+ further back along the piece.
|
|
#
|
|
# gap <= group_tolerance_ft -> A, B, C hang as ONE row, one support line.
|
|
# gap > group_tolerance_ft -> C becomes an extra "row" of its own. The
|
|
# solve then balances on a support that does
|
|
# not exist: one line goes NEGATIVE (pulled
|
|
# down) while its neighbour overloads -- on
|
|
# a real panel this read -936 lb on one
|
|
# insert and 116% of rating on the next.
|
|
#
|
|
# SET IT just above the largest stagger your yard actually leaves within a
|
|
# row. Too large merges two rows that were genuinely separate -- the two
|
|
# failure modes point opposite ways, so this number is yours to set, not a
|
|
# default to trust.'''),
|
|
|
|
('dunnage', u'''# Where the piece sits on the trailer during transportation.
|
|
#
|
|
# VALID VALUES for mode:
|
|
# null old lift approximation (transportation is still checked as a
|
|
# pick; the report labels it approximate)
|
|
# lifter_rows dunnage under the same lines as the stripping rows
|
|
# fraction two dunnage lines inset fraction*L from each end
|
|
#
|
|
# fraction is only used when mode is fraction, e.g. 0.2 means supports at
|
|
# 0.2L and 0.8L. This is plant knowledge; it is lifted up here so you can see
|
|
# and edit it without collapsing the whole knowledge tree.'''),
|
|
)
|
|
|
|
|
|
|
|
|
|
def report_dir(doc, know):
|
|
"""Where handling reports go: what was set, or beside the model."""
|
|
from vibe import designspec
|
|
return (knowledge.get(know, 'handling.report_dir') or
|
|
os.path.join(designspec.root_dir(doc), 'handling'))
|
|
|
|
|
|
def decisions(doc, know):
|
|
"""Everything in this file that is the engineer's to decide, in one place.
|
|
|
|
Every one of these lives three or four levels down the merged knowledge
|
|
tree, which is where it is read from and a hopeless place to find it.
|
|
Each is here because it was asked for by name: where the reports go,
|
|
whether they open, the prestress loss, the transfer length, how the
|
|
piece hangs at each stage, and how much stagger within a pick row still
|
|
counts as one row.
|
|
|
|
They come FIRST in the file on purpose. A config file is read from the top
|
|
and abandoned somewhere in the middle, so the part somebody opened it to
|
|
change has to sit above the part they did not.
|
|
"""
|
|
modes = {'default': knowledge.get(know, 'handling.rig_mode', 'rolling')}
|
|
for stage in knowledge.get(know, 'handling.stages', []) or []:
|
|
if stage.get('name'):
|
|
modes[stage['name']] = stage.get('rig_mode')
|
|
fast = knowledge.get(know, 'handling.fast_lifter_box', True)
|
|
fast_batch = knowledge.get(know, 'handling.fast_lifter_box_batch', False)
|
|
return {
|
|
'report_dir': report_dir(doc, know),
|
|
'open_report_when_done': knowledge.get(know, 'handling.open_report',
|
|
True),
|
|
'prestress_loss_pct': knowledge.get(know,
|
|
'handling.prestress_loss_pct',
|
|
8.0),
|
|
'transfer_length_in': knowledge.get(know,
|
|
'handling.transfer_length_in'),
|
|
'group_tolerance_ft': knowledge.get(know,
|
|
'handling.group_tolerance_ft',
|
|
1.0),
|
|
'dunnage': knowledge.get(know, 'handling.dunnage') or {},
|
|
'insert_position': {
|
|
'one_piece': 'fast' if fast else 'precise',
|
|
'batch': 'fast' if fast_batch else 'precise',
|
|
},
|
|
'rig_mode': modes,
|
|
}
|
|
|
|
|
|
def decisions_block(doc, know):
|
|
"""The decisions section as text: every value under its own comment."""
|
|
body = decisions(doc, know)
|
|
out = [DECISION_BANNER, u'', u'decisions:']
|
|
for key, note in DECISION_NOTES:
|
|
out.append(u'')
|
|
out.append(u'\n'.join(u' ' + line for line in note.split(u'\n')))
|
|
out.extend(emit({key: body[key]}, 1))
|
|
return out
|
|
|
|
|
|
def output_locations(doc, know):
|
|
"""Where each thing this tool produces is written.
|
|
|
|
Every path here is DERIVED from where the model sits. None of it is a
|
|
setting any more -- the one that was, `lifting_check_reports`, moved up
|
|
into `decisions` as `report_dir`. Two editable copies of one path is
|
|
exactly the trap this file exists not to be.
|
|
|
|
They are still listed, because "where did the specs go" is a question
|
|
people have and printing the answer beats hunting for it.
|
|
"""
|
|
from vibe import designspec
|
|
return {
|
|
'extracted_specs': designspec.sub_dir(doc, 'specs'),
|
|
'evidence': designspec.sub_dir(doc, 'evidence'),
|
|
'logs': designspec.sub_dir(doc, 'log'),
|
|
'job_knowledge_file': designspec.knowledge_path(doc),
|
|
# Resolved for THIS job's plant, not the machine-wide file -- see
|
|
# knowledge.plant_id() and the two-pass note in knowledge.load().
|
|
'plant_knowledge_file': designspec.plant_knowledge_path(
|
|
knowledge.plant_id(doc)),
|
|
}
|
|
|
|
|
|
LOCATION_NOTES = (
|
|
u'# All derived from where the model sits. Shown so you do not have to\n'
|
|
u'# hunt for them; editing them does nothing. The one path here that IS a\n'
|
|
u'# setting is report_dir, at the top of this file.'
|
|
)
|
|
|
|
|
|
HEADER = u"""# WillTools project configuration
|
|
#
|
|
# model {model}
|
|
# written {when}
|
|
#
|
|
# THIS FILE IS LIVE. It is the configuration {model} runs on, and it is the
|
|
# last word -- every setting in here beats the same setting in the layered
|
|
# files it was built from. Edit it and the next run uses what you wrote.
|
|
#
|
|
# It got that way because somebody pressed Project Config. A one-line pointer
|
|
# beside the model records where this file is:
|
|
#
|
|
# {pointer}
|
|
#
|
|
# Delete that pointer, or press Project Config again, and the model goes back
|
|
# to the layered files. Nothing else reads this path.
|
|
#
|
|
# The layers underneath, in the order they override each other. This file sits
|
|
# after all of them:
|
|
#
|
|
{layers}
|
|
#
|
|
# One consequence worth knowing. The `knowledge:` section below is the MERGED
|
|
# result, so it contains the plant's settings as values of its own. While this
|
|
# file is live, editing the plant file no longer reaches this model -- the
|
|
# copy in here wins. That is the point of pinning a configuration to a job,
|
|
# and it is also how a plant-wide fix quietly fails to arrive. When you want
|
|
# the plant layer back, clear the pointer.
|
|
"""
|
|
|
|
|
|
# Only ever appended when `layer_problems()` is non-empty. An unreadable
|
|
# layer and an empty one are OPPOSITE conditions -- one is a 255KB plant file
|
|
# silently voided (the 2026-09-01 incident that `knowledge._merge_file`'s
|
|
# marker exists for), the other is genuinely nothing -- and this file is
|
|
# where a human looks first, so the two must never render the same way.
|
|
# `# set by:` in the KNOWLEDGE section below only ever names layers that DID
|
|
# contribute a value; a layer that could not be read contributes none, which
|
|
# is exactly why it needs a banner of its own instead of relying on that line
|
|
# to mention it.
|
|
UNREADABLE_LAYER_BANNER = u'''
|
|
# =========================================================================
|
|
# ONE OR MORE KNOWLEDGE LAYERS COULD NOT BE READ
|
|
#
|
|
# Each layer named below is a file that IS on this machine, and the YAML
|
|
# reader refused it. A half-parsed layer is never merged, so NONE of that
|
|
# file was used -- every value it would have set has fallen back to
|
|
# whatever the layer underneath it says. A layer that fails to load looks
|
|
# EXACTLY like a layer that agrees with the defaults; nothing below this
|
|
# banner can tell you that happened, only this can.
|
|
#
|
|
{rows}
|
|
# Fix the file and dump again -- a key stated twice inside one mapping is
|
|
# the usual cause, and `python tools/knowledgelint.py` names it with a line
|
|
# number. The tool will not guess at what the file said.
|
|
# ========================================================================='''
|
|
|
|
|
|
def unreadable_layer_banner(problems):
|
|
"""The banner text for `layer_problems()`'s rows, or '' when there are none.
|
|
|
|
Split out from `build()` so a caller assembling its own view of the same
|
|
facts (see `layer_problems()`) does not have to reconstruct this from
|
|
scratch -- and so a test can check the wording without going through a
|
|
live `doc`.
|
|
"""
|
|
if not problems:
|
|
return u''
|
|
rows = u'#\n'.join(
|
|
u'# {0} layer -- {1}\n# {2}\n'.format(label, path, message)
|
|
for label, path, message in problems)
|
|
return UNREADABLE_LAYER_BANNER.format(rows=rows)
|
|
|
|
|
|
def _layer_status_suffix(exists, unreadable):
|
|
"""The trailing note on one layer's line in the header's layer listing.
|
|
|
|
Three facts, three different suffixes -- collapsing any two of them is
|
|
exactly the bug this exists to prevent: 'read clean' (''), 'not present'
|
|
(the file is not there, ordinary and quiet) and 'unreadable' (the file
|
|
IS there and the YAML reader refused it, which is the loud one and must
|
|
never look like either of the other two, and least of all like 'read
|
|
clean and simply set nothing').
|
|
"""
|
|
if unreadable:
|
|
return u' !! UNREADABLE !!'
|
|
if not exists:
|
|
return u' -- not present'
|
|
return u''
|
|
|
|
|
|
def build(doc, know):
|
|
"""The whole document, as text."""
|
|
from vibe import designspec
|
|
|
|
rows = _layer_rows(doc)
|
|
layer_lines = [
|
|
u'# {0:<10} {1}{2}'.format(
|
|
label, path or '(built into the tool)',
|
|
_layer_status_suffix(exists, unreadable))
|
|
for label, path, exists, _data, unreadable in rows]
|
|
|
|
body = {
|
|
'decisions': decisions(doc, know),
|
|
'project': project_info(doc, know),
|
|
'output_locations': output_locations(doc, know),
|
|
'knowledge': know,
|
|
}
|
|
|
|
text = [HEADER.format(when=datetime.datetime.now()
|
|
.strftime('%Y-%m-%d %H:%M'),
|
|
model=doc.Title,
|
|
pointer=designspec.config_pointer_path(doc),
|
|
layers=u'\n'.join(layer_lines))]
|
|
|
|
problems = [(label, path, message)
|
|
for label, path, _exists, _data, message in rows
|
|
if message]
|
|
banner = unreadable_layer_banner(problems)
|
|
if banner:
|
|
text.append(banner)
|
|
|
|
# Above everything else, because a config file is read from the top and
|
|
# abandoned in the middle.
|
|
text.append(u'')
|
|
text.append(u'\n'.join(decisions_block(doc, know)))
|
|
|
|
text.append(u'\n# What this configuration belongs to. Read for the record,'
|
|
u' not by the tool.')
|
|
text.append(u'project:')
|
|
text.append(u'\n'.join(emit(body['project'], 1)))
|
|
|
|
text.append(u'\n# Where output goes.\n#\n' + LOCATION_NOTES)
|
|
text.append(u'output_locations:')
|
|
text.append(u'\n'.join(emit(body['output_locations'], 1)))
|
|
|
|
rows = overridden(doc)
|
|
if rows:
|
|
text.append(u'\n# ---- settings more than one layer touches ----------'
|
|
u'--------------------\n#\n'
|
|
u'# Where a plant rule beats a default, or a job beats its '
|
|
u'plant. The last\n# entry on each line is the one in '
|
|
u'force.\n#')
|
|
for path, chain in rows:
|
|
text.append(u'# {0}\n# {1}'.format(
|
|
path, u' -> '.join(
|
|
u'{0}: {1}'.format(label, _scalar(value))
|
|
for label, value in chain)))
|
|
|
|
text.append(u'\n# =================================================================')
|
|
text.append(u'# KNOWLEDGE -- everything below this line is the merged configuration.')
|
|
text.append(u'#')
|
|
text.append(u'# This whole section is one `knowledge:` block, so most editors let you')
|
|
text.append(u'# collapse it (click the triangle / fold marker next to `knowledge:`).')
|
|
text.append(u'# You almost never need to edit anything here. The values the engineer')
|
|
text.append(u'# actually decides are in the `decisions:` section at the top of this')
|
|
text.append(u'# file; anything also named there is overruled by the copy up there.')
|
|
text.append(u'#')
|
|
text.append(u'# The "set by" comments say which underlying layer each value came from,')
|
|
text.append(u'# so you can tell a deliberate plant rule from a default nobody has')
|
|
text.append(u'# looked at.')
|
|
text.append(u'\nknowledge:')
|
|
text.append(_knowledge_body(doc, know, indent=1))
|
|
return u'\n'.join(text) + u'\n'
|
|
|
|
|
|
def _knowledge_body(doc, know, indent):
|
|
"""The knowledge tree, section by section, each one labelled and sourced.
|
|
|
|
Emitting the whole tree in one go would be shorter and much harder to
|
|
read: a merged configuration is several hundred lines and every one of
|
|
them looks equally important until something says otherwise.
|
|
"""
|
|
where = origins(doc)
|
|
pad = ' ' * indent
|
|
out = []
|
|
for name in sections(know):
|
|
note = SECTION_NOTES.get(name)
|
|
layers = sorted(set(
|
|
where[path] for path in _paths({name: know[name]})
|
|
if path in where))
|
|
out.append(u'')
|
|
if note:
|
|
out.append(u'{0}# {1}'.format(pad, note))
|
|
if layers:
|
|
out.append(u'{0}# set by: {1}'.format(pad, ', '.join(layers)))
|
|
out.extend(emit({name: know[name]}, indent))
|
|
return u'\n'.join(out)
|