will
52d94303bc
ci / test (push) Has been cancelled
Public demo snapshot of an in-house pyRevit extension for precast concrete shop-ticket QC (37 checks / 46 checklist rows) and a standalone panel handling solver (PCI Chapter 8). - No client, job, plant or product data: knowledge layers ship as structural templates only. - Ticket QC's model-read escalation targets a LAN model server endpoint (configurable), not a cloud provider. - Built from the private development repo via tools/make_dist.py.
792 lines
30 KiB
Python
792 lines
30 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""A small, hand-rolled YAML subset: write it, read it back, nothing else.
|
|
|
|
Two things this is not: a general YAML library, and a wrapper around one.
|
|
Neither exists for free here -- PyYAML is a C extension, not available under
|
|
IronPython 2.7, and the one YAML reader this extension already had
|
|
(`pyrevit.coreutils.yaml`) only exists inside a running Revit/pyRevit process,
|
|
which is exactly the environment a unit test does not have.
|
|
|
|
So this module writes ONE well-defined, restricted shape of YAML -- block
|
|
mappings, block and flow lists, quoted/plain scalars, full-line comments --
|
|
and reads back exactly that shape. It has no Revit dependency and no
|
|
third-party dependency, so it is the same code under IronPython 2.7 inside
|
|
Revit and under CPython in a test. `configdump.py` (the project-config dump)
|
|
and `designspec.py` / `specformat.py` (the design spec) both write through
|
|
`emit()` and read through `parse()`; neither has its own YAML code.
|
|
|
|
The contract `parse()` relies on: every document `emit()` produces is a tree
|
|
of dicts, lists and scalars, indented two spaces per level, with a list
|
|
item's dash occupying exactly one indent unit (`'- '` is two characters, the
|
|
same width as `' '`) so a list item's own fields line up under it. That is
|
|
not a guess about YAML in general -- it is what `emit()` itself does, checked
|
|
against its own output below.
|
|
"""
|
|
# pylint: disable=invalid-name
|
|
import re
|
|
|
|
from vibe.compat import string_types, text_type
|
|
|
|
|
|
_PLAIN = re.compile(r'^[A-Za-z0-9_][A-Za-z0-9_ ./+-]*$')
|
|
_RESERVED = ('true', 'false', 'null', 'yes', 'no', 'on', 'off', '~')
|
|
_NUMBER_LOOKING = re.compile(r'^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$')
|
|
|
|
|
|
def scalar(value):
|
|
"""One value as it is written on the right of a `key:`."""
|
|
if value is None:
|
|
return 'null'
|
|
if value is True:
|
|
return 'true'
|
|
if value is False:
|
|
return 'false'
|
|
if isinstance(value, float):
|
|
text = repr(value)
|
|
return text[:-2] if text.endswith('.0') else text
|
|
if not isinstance(value, string_types):
|
|
return str(value)
|
|
|
|
text = text_type(value)
|
|
if not text:
|
|
return "''"
|
|
if text.lower() in _RESERVED or not _PLAIN.match(text) \
|
|
or text != text.strip() or text.endswith(':'):
|
|
return "'" + text.replace("'", "''") + "'"
|
|
# A bare number-looking string has to be quoted or it reads back as a
|
|
# number, and a part number like 4.10 would come back as 4.1.
|
|
if _NUMBER_LOOKING.match(text):
|
|
return "'" + text + "'"
|
|
return text
|
|
|
|
|
|
def is_flat(node):
|
|
"""A list of plain scalars, short enough to sit on one line."""
|
|
if not isinstance(node, list) or not node:
|
|
return False
|
|
if any(isinstance(v, (dict, list)) for v in node):
|
|
return False
|
|
return sum(len(scalar(v)) + 2 for v in node) <= 66
|
|
|
|
|
|
def emit(node, indent=0):
|
|
"""A dict/list/scalar tree as YAML lines. Deterministic key order."""
|
|
pad = ' ' * indent
|
|
out = []
|
|
|
|
if isinstance(node, dict):
|
|
if not node:
|
|
return [pad + '{}']
|
|
for key in sorted(node):
|
|
value = node[key]
|
|
label = pad + scalar(key) + ':'
|
|
if isinstance(value, dict) and value:
|
|
out.append(label)
|
|
out.extend(emit(value, indent + 1))
|
|
elif isinstance(value, list) and value:
|
|
if is_flat(value):
|
|
out.append('{0} [{1}]'.format(
|
|
label, ', '.join(scalar(v) for v in value)))
|
|
else:
|
|
out.append(label)
|
|
out.extend(emit(value, indent + 1))
|
|
elif isinstance(value, (dict, list)):
|
|
out.append('{0} {1}'.format(label,
|
|
'{}' if isinstance(value, dict)
|
|
else '[]'))
|
|
else:
|
|
out.append('{0} {1}'.format(label, scalar(value)))
|
|
return out
|
|
|
|
if isinstance(node, list):
|
|
for item in node:
|
|
if isinstance(item, dict) and item:
|
|
lines = emit(item, indent + 1)
|
|
# The first key rides on the dash, the rest line up under it.
|
|
out.append(pad + '- ' + lines[0].strip())
|
|
out.extend(lines[1:])
|
|
elif isinstance(item, list) and item:
|
|
lines = emit(item, indent + 1)
|
|
out.append(pad + '-')
|
|
out.extend(lines)
|
|
else:
|
|
out.append(pad + '- ' + scalar(item))
|
|
return out
|
|
|
|
return [pad + scalar(node)]
|
|
|
|
|
|
# ------------------------------------------------------------------- parse
|
|
|
|
class YamlKitError(ValueError):
|
|
"""A document this parser cannot read, said out loud instead of cut short.
|
|
|
|
A `ValueError` subclass so a caller that already guards a parse with
|
|
`except ValueError` keeps working, and its own class so one that wants
|
|
to name this failure can.
|
|
|
|
WHY THIS EXISTS AT ALL -- the lesson, kept where the code is.
|
|
|
|
A parser that stops early without saying so is worse than one that
|
|
raises, and this module has now proved it repeatedly. Every one of
|
|
these is ordinary YAML, none of them is anything `emit()` writes, and
|
|
each of them silently swallowed the rest of a real file:
|
|
|
|
* `knowledge/catalogue.yaml` carried one `note: >-` two thirds of the
|
|
way down, and every line under it -- the whole rebar branch -- was
|
|
invisible while the file "parsed fine" (fixed: `_is_block_scalar`);
|
|
* a job knowledge file carried `parts: [P76, P89,` wrapping onto the
|
|
next line, and the three sections below it were simply absent;
|
|
* the same file wrote a block sequence at its key's own indentation
|
|
(`notes:` then `- ...` in the same column);
|
|
* it wrapped a long list item's text onto a second, indented line;
|
|
* a plant file put a `# comment` on the end of a `key:` line.
|
|
|
|
16 top-level keys in that job file, 13 in what `parse()` returned.
|
|
|
|
The second one cost a real press. The config editor cross-checks its
|
|
tree against the one the Revit-side reader builds, key by key, and the
|
|
key it happened to be writing was `snapshot.detached_dir` -- so a
|
|
WHOLE-FILE failure was reported as "the two YAML readers disagree about
|
|
snapshot.detached_dir", a sentence about an innocent key that sent the
|
|
reader to the wrong file entirely. Every key past the cut would have
|
|
produced the same message and the same wrong destination.
|
|
|
|
So the rule for anything added to this parser: a construct it cannot
|
|
read raises. It never returns the part it managed to get through.
|
|
"""
|
|
|
|
|
|
def parse(text):
|
|
"""The inverse of `emit()`. Reads back exactly the shape it produces.
|
|
|
|
Comments (any line whose stripped text starts with '#') and blank lines
|
|
are ignored wherever they appear -- a spec file is meant to carry a
|
|
comment above every section, and this is what makes that free. The
|
|
document is assumed to be one mapping at the top, which is true of every
|
|
file this module writes (a knowledge dump, a design spec).
|
|
|
|
Beyond what `emit()` writes, `_logical_lines` folds the hand-written
|
|
shapes a knowledge file really contains -- a wrapped flow sequence, a
|
|
trailing comment, a plain scalar continued on the next line -- before
|
|
any of the block grammar runs. Every one of them used to truncate the
|
|
document in silence; see `YamlKitError`.
|
|
|
|
Raises `YamlKitError` on a flow sequence that is never closed.
|
|
"""
|
|
lines = _logical_lines(text)
|
|
if not lines:
|
|
return {}
|
|
top_indent = lines[0][0]
|
|
value, _idx = _parse_dict(lines, 0, top_indent)
|
|
return value
|
|
|
|
|
|
def _logical_lines(text):
|
|
"""Physical lines folded into the (level, text) pairs the grammar reads.
|
|
|
|
Blank lines and whole-line comments are dropped wherever they appear --
|
|
a spec file is meant to carry a comment above every section, and this
|
|
is what makes that free. What is left is one entry per LOGICAL line,
|
|
which is not always one physical line: see `_join_flow` and
|
|
`_fold_plain`.
|
|
|
|
The one place none of this applies is the body of a block scalar, where
|
|
a '#' is text and an indented line is content rather than structure.
|
|
Those lines are passed through verbatim for `_parse_dict` to gather.
|
|
"""
|
|
raw = (text or '').split('\n')
|
|
total = len(raw)
|
|
out = []
|
|
index = 0
|
|
block_level = None
|
|
while index < total:
|
|
line = raw[index]
|
|
number = index + 1
|
|
index += 1
|
|
stripped = line.strip()
|
|
if not stripped or stripped.startswith('#'):
|
|
continue
|
|
# emit() always indents two spaces per level (a list item's dash is
|
|
# the same width as one level, see the module docstring), so integer
|
|
# division recovers the level exactly -- no half-levels exist here.
|
|
level = (len(line) - len(line.lstrip(' '))) // 2
|
|
if block_level is not None:
|
|
if level > block_level:
|
|
out.append((level, stripped))
|
|
continue
|
|
block_level = None
|
|
body = _strip_comment(stripped)
|
|
if not body:
|
|
continue
|
|
offset = _flow_value_offset(body)
|
|
if offset >= 0:
|
|
body, index = _join_flow(raw, index, body, offset, number)
|
|
elif _plain_value(body):
|
|
body, index = _fold_plain(raw, index, body, level)
|
|
if _is_block_scalar(_line_value(body) or ''):
|
|
block_level = level
|
|
out.append((level, body))
|
|
return out
|
|
|
|
|
|
def _line_value(text):
|
|
"""The value half of a line, read the way the block grammar reads it.
|
|
|
|
`None` when the line has no value half at all (a bare `-`, or a line
|
|
with no colon). Mirrors `_parse_dict`'s `partition(':')` and
|
|
`_parse_list`'s `_is_mapping_line` exactly, so the pre-pass can never
|
|
disagree with the parse that follows about what a line's value is.
|
|
"""
|
|
if text == '-':
|
|
return None
|
|
if text[:2] == '- ':
|
|
remainder = text[2:]
|
|
if _is_mapping_line(remainder):
|
|
return remainder.partition(':')[2].strip()
|
|
return remainder.strip()
|
|
_key, sep, rest = text.partition(':')
|
|
if not sep:
|
|
return None
|
|
return rest.strip()
|
|
|
|
|
|
def _plain_value(text):
|
|
"""True when this line's value is a plain scalar that could continue.
|
|
|
|
Not a block-scalar indicator (`_parse_dict` gathers those itself), not
|
|
a flow collection (`_join_flow` has already dealt with those), and not
|
|
empty (an empty value means the value is the indented block below).
|
|
"""
|
|
value = _line_value(text)
|
|
return bool(value) and not _is_block_scalar(value)
|
|
|
|
|
|
def _fold_plain(raw, index, body, level):
|
|
"""A plain scalar continued on the following, more-indented lines.
|
|
|
|
YAML folds those into the one scalar with a space, and a hand-written
|
|
note that ran long is exactly where it happens. This parser used to do
|
|
something far worse than misread it: an indented line after a `key:
|
|
value` matches no branch of `_parse_dict`'s loop, so it BROKE the loop
|
|
and every enclosing one, and the document ended there.
|
|
|
|
A continuation stops at the first line that is not more indented, at a
|
|
blank or comment line, and at any line that is itself a list item or a
|
|
`key: value` -- that last guard is what keeps the second field of a
|
|
`- key: value` list item a field instead of swallowing it into the
|
|
first one's text.
|
|
"""
|
|
total = len(raw)
|
|
while index < total:
|
|
line = raw[index]
|
|
stripped = line.strip()
|
|
if not stripped or stripped.startswith('#'):
|
|
break
|
|
if (len(line) - len(line.lstrip(' '))) // 2 <= level:
|
|
break
|
|
text = _strip_comment(stripped)
|
|
if not text or _is_list_line(text) or _is_mapping_line(text):
|
|
break
|
|
body = body + ' ' + text
|
|
index += 1
|
|
return body, index
|
|
|
|
|
|
def _strip_comment(text):
|
|
"""One line with an unquoted trailing `# comment` removed.
|
|
|
|
A '#' only starts a comment where YAML says it does -- at the start of
|
|
the line or after whitespace, and never inside a quoted scalar -- so
|
|
`sheet_re: ^WP#\\d+` keeps its hash and `dims: [3", 3"]` keeps its inch
|
|
marks. `emit()` quotes any value containing a '#' (the plain-scalar
|
|
pattern excludes it), so nothing this module writes is touched here;
|
|
this is entirely for the files people edit by hand, where a comment on
|
|
the end of a `key:` line used to end the document.
|
|
"""
|
|
return _flow_scan(text, 0, 0, '')[0]
|
|
|
|
|
|
def _join_flow(raw_lines, index, first, offset, number):
|
|
"""One logical line from a flow sequence that wraps. Returns (text, idx).
|
|
|
|
`emit()` never writes one -- `is_flat` keeps a flow list to 66
|
|
characters on a single line -- so this shape only ever arrives by a
|
|
human hand-editing the file, which is exactly why it has to be read.
|
|
The continuation lines are joined with a single space, the way YAML
|
|
folds them, and their own indentation is discarded: the level of the
|
|
logical line is the level of the line the `[` opened on.
|
|
|
|
Raises `YamlKitError` when the bracket is never closed. THAT IS THE
|
|
DELIBERATE CHOICE, not an oversight: running off the end of the file
|
|
and returning the mapping built so far is precisely the silent
|
|
truncation this whole path exists to end. An unclosed `[` is a broken
|
|
file and the caller has to be told, not handed two thirds of a tree.
|
|
"""
|
|
body, depth, quote = _flow_scan(first, offset, 0, '')
|
|
parts = [body]
|
|
total = len(raw_lines)
|
|
while depth > 0:
|
|
if index >= total:
|
|
raise YamlKitError(
|
|
'unterminated flow sequence: the "[" opened on line {0} is '
|
|
'never closed, so the rest of the document cannot be read. '
|
|
'Nothing was returned -- a partial tree is how this bug hid '
|
|
'the last time. The line: {1!r}'.format(number, first[:70]))
|
|
nxt = raw_lines[index].strip()
|
|
index += 1
|
|
# A blank line or a whole-line comment inside a flow sequence is
|
|
# skipped, the same as anywhere else -- unless a quoted scalar is
|
|
# still open across the line break, where a '#' is just text.
|
|
if not quote and (not nxt or nxt.startswith('#')):
|
|
continue
|
|
body, depth, quote = _flow_scan(nxt, 0, depth, quote)
|
|
parts.append(body)
|
|
return ' '.join([p for p in parts if p]), index
|
|
|
|
|
|
def _flow_value_offset(text):
|
|
"""Where a flow collection opens on this line, or -1 for none.
|
|
|
|
Only a value that BEGINS with `[` (or `{`) opens one. YAML lets a plain
|
|
scalar carry a bracket in the middle of itself -- `note: see [1` is the
|
|
string "see [1" and nothing is unclosed about it -- so counting every
|
|
bracket on every line would swallow the rest of the document hunting a
|
|
`]` that was never a bracket at all.
|
|
|
|
`{` counts as an opener the same way `[` does -- see `_parse_flow_map`.
|
|
Joining it matters for the same reason: an unjoined one that wrapped
|
|
would end the enclosing mapping and lose the rest of the file, and
|
|
losing one value beats losing the document.
|
|
"""
|
|
body = text
|
|
offset = 0
|
|
if body[:2] == '- ':
|
|
rest = body[2:].lstrip()
|
|
offset = len(body) - len(rest)
|
|
body = rest
|
|
elif body == '-':
|
|
return -1
|
|
# `x[:1] in '[{'` is True for the empty string -- guard the head first.
|
|
if body[:1] and body[0] in '[{':
|
|
return offset
|
|
# `key: [...]`. Partitioned exactly the way _parse_dict partitions it,
|
|
# so this can never disagree with the parse that follows.
|
|
key, sep, rest = body.partition(':')
|
|
if not sep:
|
|
return -1
|
|
value = rest.lstrip()
|
|
if not value or value[0] not in '[{':
|
|
return -1
|
|
return offset + len(key) + 1 + (len(rest) - len(value))
|
|
|
|
|
|
# A quote character only OPENS a quoted scalar where a scalar can start:
|
|
# at the head of the value, or after a separator. Anywhere else it is
|
|
# ordinary text, which YAML agrees with and this project needs -- the
|
|
# shipped catalogue writes `dims: [3", 3", 3"]`, where every `"` is an inch
|
|
# mark. Toggling on those turns half the line into "a quoted string" and
|
|
# hides the closing bracket, which is the truncation bug again wearing a
|
|
# different hat. `it's` in an unquoted scalar is the same case.
|
|
_QUOTE_OPENS = ' \t[{,:'
|
|
|
|
|
|
def _opens_quote(ch, prev):
|
|
return (ch == "'" or ch == '"') and (prev == '' or prev in _QUOTE_OPENS)
|
|
|
|
|
|
def _flow_scan(text, start, depth, quote):
|
|
"""Walk one physical line of a flow collection.
|
|
|
|
Returns (body, depth, quote): `body` is the line with an unquoted
|
|
trailing comment removed, `depth` the running bracket depth, and
|
|
`quote` the quote character still open at the end of the line ('' when
|
|
none). Everything before `start` is left untouched -- that is the
|
|
`key:` half of the line, which the block grammar still has to see.
|
|
|
|
Quotes are tracked so a value that contains a bracket or a comma
|
|
("'P76, [spare]'") neither closes the sequence early nor splits, and so
|
|
a '#' inside a quoted scalar stays part of the scalar instead of
|
|
starting a comment. A '#' only starts a comment where YAML says it
|
|
does: at the start of the line or after whitespace.
|
|
"""
|
|
i = start
|
|
n = len(text)
|
|
while i < n:
|
|
ch = text[i]
|
|
if quote:
|
|
if ch == quote:
|
|
if quote == "'" and text[i + 1:i + 2] == "'":
|
|
i += 2
|
|
continue
|
|
quote = ''
|
|
elif quote == '"' and ch == '\\':
|
|
i += 2
|
|
continue
|
|
i += 1
|
|
continue
|
|
if _opens_quote(ch, text[i - 1] if i else ''):
|
|
quote = ch
|
|
i += 1
|
|
continue
|
|
if ch == '#' and (i == 0 or text[i - 1] in ' \t'):
|
|
return text[:i].rstrip(), depth, ''
|
|
if ch == '[' or ch == '{':
|
|
depth += 1
|
|
elif ch == ']' or ch == '}':
|
|
if depth > 0:
|
|
depth -= 1
|
|
i += 1
|
|
return text.rstrip(), depth, quote
|
|
|
|
|
|
def loads(text):
|
|
"""Alias for parse(), read as "load a string" the way json.loads reads."""
|
|
return parse(text)
|
|
|
|
|
|
def _parse_dict(lines, idx, indent, first=None):
|
|
result = {}
|
|
n = len(lines)
|
|
pending = [first] if first is not None else []
|
|
while True:
|
|
if pending:
|
|
text = pending.pop(0)
|
|
consume = False
|
|
else:
|
|
if idx >= n:
|
|
break
|
|
cur_indent, text = lines[idx]
|
|
if cur_indent != indent or _is_list_line(text):
|
|
break
|
|
consume = True
|
|
|
|
key, sep, rest = text.partition(':')
|
|
if not sep:
|
|
# Malformed line for this grammar -- skip rather than raise, so
|
|
# one stray line does not take the whole file down.
|
|
if consume:
|
|
idx += 1
|
|
continue
|
|
key = _parse_key(key)
|
|
rest = rest.strip()
|
|
if consume:
|
|
idx += 1
|
|
|
|
if _is_block_scalar(rest):
|
|
# `key: >-` and friends. emit() never writes one, so for a long
|
|
# time this parser did not read one either -- and a file with a
|
|
# block scalar in it did not fail, it TRUNCATED: the indicator
|
|
# was taken as the value and every more-indented line after it
|
|
# ended the enclosing mapping. `knowledge/catalogue.yaml` had one
|
|
# `note: >-` two thirds of the way down, and everything below it
|
|
# -- the whole rebar branch -- was invisible to every host test
|
|
# that read the shipped file, while the file itself parsed
|
|
# "fine". Hand-written knowledge files are exactly where a folded
|
|
# scalar gets used, so reading them is not optional.
|
|
#
|
|
# The subset: folded (`>`) joins with spaces, literal (`|`) with
|
|
# newlines. Chomping is read, because the default is NOT the
|
|
# obvious one and the difference is a whole reader
|
|
# disagreement: a bare `>` CLIPS, which keeps exactly one
|
|
# trailing newline, and only `>-` strips it. A `+` (keep all)
|
|
# is read as clip -- parse() has already dropped the blank
|
|
# lines that would tell us how many, and one is the nearest
|
|
# true answer available. What this cannot see, for the same
|
|
# reason, is a blank line or a `#` line INSIDE a block, so a
|
|
# literal block loses its paragraph breaks. Fine for a sentence
|
|
# in a knowledge file, which is all anyone writes here.
|
|
text_lines = []
|
|
while idx < n and lines[idx][0] > indent:
|
|
text_lines.append(lines[idx][1])
|
|
idx += 1
|
|
joiner = u'\n' if rest[:1] == '|' else u' '
|
|
body = joiner.join(text_lines)
|
|
if body and '-' not in rest[1:]:
|
|
body += u'\n'
|
|
result[key] = body
|
|
elif rest == '':
|
|
if idx < n and lines[idx][0] > indent:
|
|
child_indent = lines[idx][0]
|
|
if _is_list_line(lines[idx][1]):
|
|
child, idx = _parse_list(lines, idx, child_indent)
|
|
else:
|
|
child, idx = _parse_dict(lines, idx, child_indent)
|
|
result[key] = child
|
|
elif idx < n and lines[idx][0] == indent \
|
|
and _is_list_line(lines[idx][1]):
|
|
# A block sequence written at its KEY'S OWN indentation:
|
|
#
|
|
# notes:
|
|
# - the first note
|
|
# - the second
|
|
#
|
|
# Valid YAML, never what emit() writes (it indents a list
|
|
# one level in), and the THIRD construct found truncating a
|
|
# hand-edited file: the key took None, then the `- ` line at
|
|
# the mapping's own indent broke the loop and every enclosing
|
|
# one with it, so the rest of the document vanished. There is
|
|
# no ambiguity to weigh -- a bare `- ` item cannot be a
|
|
# sibling of mapping keys, so a list here belongs to the key
|
|
# immediately above it and to nothing else.
|
|
child, idx = _parse_list(lines, idx, indent)
|
|
result[key] = child
|
|
else:
|
|
result[key] = None
|
|
elif rest == '{}':
|
|
result[key] = {}
|
|
elif rest == '[]':
|
|
result[key] = []
|
|
elif _is_flow(rest):
|
|
result[key] = _parse_flow(rest)
|
|
else:
|
|
result[key] = _parse_scalar(rest)
|
|
return result, idx
|
|
|
|
|
|
def _parse_list(lines, idx, indent):
|
|
result = []
|
|
n = len(lines)
|
|
while idx < n:
|
|
cur_indent, text = lines[idx]
|
|
if cur_indent != indent or not _is_list_line(text):
|
|
break
|
|
if text == '-':
|
|
idx += 1
|
|
if idx < n and lines[idx][0] > indent:
|
|
child_indent = lines[idx][0]
|
|
if _is_list_line(lines[idx][1]):
|
|
child, idx = _parse_list(lines, idx, child_indent)
|
|
else:
|
|
child, idx = _parse_dict(lines, idx, child_indent)
|
|
result.append(child)
|
|
else:
|
|
result.append(None)
|
|
continue
|
|
|
|
remainder = text[2:]
|
|
idx += 1
|
|
if remainder == '{}':
|
|
result.append({})
|
|
elif _is_flow(remainder):
|
|
result.append(_parse_flow(remainder))
|
|
elif _is_mapping_line(remainder):
|
|
item, idx = _parse_dict(lines, idx, indent + 1, first=remainder)
|
|
result.append(item)
|
|
else:
|
|
result.append(_parse_scalar(remainder))
|
|
return result, idx
|
|
|
|
|
|
_BLOCK_SCALAR = re.compile(r'^[|>][+-]?\d*[+-]?$')
|
|
|
|
|
|
def _is_block_scalar(text):
|
|
"""`>`, `>-`, `|`, `|+`, `|2-` -- the whole of the indicator grammar."""
|
|
return bool(text) and bool(_BLOCK_SCALAR.match(text))
|
|
|
|
|
|
def _is_list_line(text):
|
|
return text == '-' or text.startswith('- ')
|
|
|
|
|
|
def _is_mapping_line(text):
|
|
"""'key: value' vs a scalar that happens to contain a colon.
|
|
|
|
Every key `emit()` writes is a plain field name -- the scalar() rules
|
|
quote anything else, including a string ending in ':' -- so a colon
|
|
following an identifier, unquoted, is a key/value pair and nothing else
|
|
reaching this parser can look like one.
|
|
"""
|
|
if text[:1] == "'":
|
|
return False
|
|
if ':' not in text:
|
|
return False
|
|
key = text.split(':', 1)[0]
|
|
return bool(re.match(r'^[A-Za-z0-9_]+$', key))
|
|
|
|
|
|
def _is_flow(text):
|
|
"""A complete `[...]` or `{...}` -- one line, by the time it gets here."""
|
|
return ((text[:1] == '[' and text[-1:] == ']') or
|
|
(text[:1] == '{' and text[-1:] == '}')) and len(text) >= 2
|
|
|
|
|
|
def _parse_flow(text):
|
|
"""A flow collection: `[a, b]` as a list, `{a: 1, b: 2}` as a dict."""
|
|
if text[:1] == '{':
|
|
return _parse_flow_map(text)
|
|
return _parse_flow_list(text)
|
|
|
|
|
|
def _split_flow(inner):
|
|
"""The top-level items of a flow collection's interior.
|
|
|
|
Splits on commas that are neither inside a quoted scalar nor inside a
|
|
nested collection: a comma in `[[1, 2], [3]]` or in `'P76, spare'` is
|
|
not a separator of the outer collection, and a bracket inside a quoted
|
|
scalar does not open one.
|
|
"""
|
|
parts = []
|
|
current = []
|
|
quote = ''
|
|
depth = 0
|
|
i = 0
|
|
while i < len(inner):
|
|
ch = inner[i]
|
|
if quote:
|
|
if ch == quote:
|
|
if quote == "'" and inner[i + 1:i + 2] == "'":
|
|
current.append("'")
|
|
i += 2
|
|
continue
|
|
quote = ''
|
|
current.append(ch)
|
|
i += 1
|
|
continue
|
|
if _opens_quote(ch, inner[i - 1] if i else ''):
|
|
quote = ch
|
|
current.append(ch)
|
|
i += 1
|
|
continue
|
|
if ch == '[' or ch == '{':
|
|
depth += 1
|
|
elif ch == ']' or ch == '}':
|
|
if depth > 0:
|
|
depth -= 1
|
|
elif ch == ',' and not depth:
|
|
parts.append(''.join(current).strip())
|
|
current = []
|
|
i += 1
|
|
continue
|
|
current.append(ch)
|
|
i += 1
|
|
tail = ''.join(current).strip()
|
|
if tail:
|
|
parts.append(tail)
|
|
return parts
|
|
|
|
|
|
def _parse_flow_list(text):
|
|
"""`[a, b, 'c, d']` as a list. Complete on one line by now.
|
|
|
|
A wrapped sequence has already been folded onto one line by
|
|
`_join_flow`, so this only ever sees a closed `[...]`.
|
|
"""
|
|
inner = text[1:-1].strip()
|
|
if not inner:
|
|
return []
|
|
return [_flow_item(p) for p in _split_flow(inner)]
|
|
|
|
|
|
def _parse_flow_map(text):
|
|
"""`{a: 1, b: 2}` as a dict.
|
|
|
|
THIS USED TO COME BACK AS THE RAW STRING, which is the module's other
|
|
silent-wrong-answer shape: nothing failed, the value simply was not a
|
|
mapping, and every key under it was missing from the tree without a
|
|
word. A real job knowledge file writes 29 of them
|
|
(`bar_notes: {SR356: OPPOSING PAIRS}`), and the config editor's
|
|
whole-file coverage check counts paths -- so those 29 read as 41
|
|
settings the editor could not see and refused to write beside.
|
|
|
|
An item with no colon is not a mapping entry and is dropped rather than
|
|
guessed at; `_parse_dict` treats a colon-less line the same way.
|
|
"""
|
|
inner = text[1:-1].strip()
|
|
result = {}
|
|
if not inner:
|
|
return result
|
|
for part in _split_flow(inner):
|
|
key, sep, rest = _split_flow_pair(part)
|
|
if not sep:
|
|
continue
|
|
result[_parse_key(key)] = _flow_item(rest.strip())
|
|
return result
|
|
|
|
|
|
def _split_flow_pair(part):
|
|
"""One `key: value` of a flow mapping, split at the first free colon."""
|
|
quote = ''
|
|
i = 0
|
|
while i < len(part):
|
|
ch = part[i]
|
|
if quote:
|
|
if ch == quote:
|
|
if quote == "'" and part[i + 1:i + 2] == "'":
|
|
i += 2
|
|
continue
|
|
quote = ''
|
|
i += 1
|
|
continue
|
|
if _opens_quote(ch, part[i - 1] if i else ''):
|
|
quote = ch
|
|
i += 1
|
|
continue
|
|
if ch == ':':
|
|
return part[:i], ':', part[i + 1:]
|
|
i += 1
|
|
return part, '', ''
|
|
|
|
|
|
def _flow_item(text):
|
|
"""One item of a flow collection: nested collection, or scalar."""
|
|
if text == '{}':
|
|
return {}
|
|
if text == '[]':
|
|
return []
|
|
if _is_flow(text):
|
|
return _parse_flow(text)
|
|
return _parse_scalar(text)
|
|
|
|
|
|
def _unquote(text):
|
|
"""A quoted scalar's content, or None when the text is not quoted.
|
|
|
|
Single quotes are what `emit()` writes and `''` is the escape. DOUBLE
|
|
quotes are read too, for the same reason everything else in this
|
|
pre-pass is: a person writing a knowledge file by hand uses them, and
|
|
a value the two readers disagree about is a value the config editor
|
|
refuses to write beside. The escape handled is the one that matters
|
|
for this subset -- a backslash before a quote or a backslash -- not the
|
|
whole of YAML's double-quoted grammar (\\u, \\n and friends are left
|
|
alone rather than half-implemented).
|
|
|
|
Inch marks are safe: `3"` neither starts nor ends the way this needs.
|
|
"""
|
|
if len(text) < 2:
|
|
return None
|
|
if text[0] == "'" and text[-1] == "'":
|
|
return text[1:-1].replace("''", "'")
|
|
if text[0] == '"' and text[-1] == '"':
|
|
return text[1:-1].replace('\\"', '"').replace('\\\\', '\\')
|
|
return None
|
|
|
|
|
|
def _parse_key(text):
|
|
text = text.strip()
|
|
inner = _unquote(text)
|
|
return text if inner is None else inner
|
|
|
|
|
|
def _parse_scalar(text):
|
|
text = text.strip()
|
|
inner = _unquote(text)
|
|
if inner is not None:
|
|
return inner
|
|
if text == 'null':
|
|
return None
|
|
if text == 'true':
|
|
return True
|
|
if text == 'false':
|
|
return False
|
|
if re.match(r'^[+-]?\d+$', text):
|
|
return int(text)
|
|
if re.match(r'^[+-]?(\d+\.\d*|\.\d+|\d+)([eE][+-]?\d+)?$', text) and \
|
|
('.' in text or 'e' in text or 'E' in text):
|
|
try:
|
|
return float(text)
|
|
except ValueError:
|
|
pass
|
|
return text
|