will
52d94303bc
ci / test (push) Has been cancelled
Public demo snapshot of an in-house pyRevit extension for precast concrete shop-ticket QC (37 checks / 46 checklist rows) and a standalone panel handling solver (PCI Chapter 8). - No client, job, plant or product data: knowledge layers ship as structural templates only. - Ticket QC's model-read escalation targets a LAN model server endpoint (configurable), not a cloud provider. - Built from the private development repo via tools/make_dist.py.
715 lines
28 KiB
Python
715 lines
28 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""Read one shop-ticket PDF into a structure a revision diff can reason about.
|
||
|
||
WHY THIS MODULE EXISTS
|
||
|
||
A ticket is not a page of prose. It is a title block, a bill of material, three
|
||
handling diagrams, a set of note blocks, and a drawing carrying dimension
|
||
chains and part callouts. Those are the things a precast engineer names when
|
||
he says what changed. A diff that works on lines of text says "line 47
|
||
differs"; a diff that works on these says "the dunnage diagram's near-end
|
||
inset went from 4'-9" to 5'-0"". Only the second one gets read.
|
||
|
||
So extraction is region-aware, and every region is found by an ANCHOR, never
|
||
by an absolute coordinate. Measured reason: WP199 puts its handling diagrams
|
||
at x=819 and WP219 puts them at x=910. A coordinate-keyed extractor would have
|
||
silently returned nothing for half the set and reported "no changes".
|
||
|
||
WHAT THIS MODULE REFUSES TO DO
|
||
|
||
It does not guess. Four PDF text-layer failure modes were measured on this
|
||
project's other source documents, and each one produces a CONFIDENT WRONG
|
||
NUMBER rather than an error:
|
||
|
||
1. stacked fractions collapsing ("1316"" for both 13/16" and 1 3/16")
|
||
2. cipher-shifted subset fonts (characters mapped 29 code points down)
|
||
3. stamps invisible to text search
|
||
4. retired text sitting underneath current text, returned first
|
||
|
||
A diff is uniquely exposed to these, because a misread on ONE SIDE ONLY
|
||
invents a change that never happened. So `hazards()` scans for all four and
|
||
the diff downgrades or drops anything they touch. Measured on this ticket set:
|
||
modes 1 and 2 are absent (fractions come through as "1/2", no non-ASCII byte
|
||
in 46,079 spans), mode 4 is present (345 exactly-colocated duplicate spans,
|
||
2 colocated pairs with DIFFERENT text). That measurement is what licenses
|
||
reading these particular numbers off the text layer; it is not a general
|
||
licence, which is why the scan runs on every file every time.
|
||
|
||
CPython only -- pymupdf is not available to IronPython, so the Revit-side code
|
||
must not import this. The comparison half lives in ticketdelta.py and is pure.
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# dimension parsing
|
||
# --------------------------------------------------------------------------
|
||
|
||
# A ticket writes the same length two ways depending on which family drew it:
|
||
# the bed plan prints 1' - 0", the handling diagrams print 1'-0". Comparing the
|
||
# strings would report a change on every ticket. Compare the VALUE, show the
|
||
# RAW TEXT.
|
||
_DIM_RE = re.compile(
|
||
r'^(?:(\d+)\'\s*-?\s*)?' # feet
|
||
r'(?:(\d+)(?!/))?' # whole inches (not the numerator of a fraction)
|
||
r'\s*(?:(\d+)/(\d+))?' # fractional inches
|
||
r'"$'
|
||
)
|
||
|
||
|
||
def parse_dim(text):
|
||
"""Feet-and-inches string -> inches as a float, or None if not a dimension.
|
||
|
||
Accepts 22'-11", 22' - 11", 2' - 5 1/2", 11 1/2", 8", 3/4".
|
||
Returns None for anything else -- including things that merely contain a
|
||
dimension, such as (5) @ 1'-6" = 7'-6", which is a note, not a dimension.
|
||
"""
|
||
if not text:
|
||
return None
|
||
t = text.strip().replace(u'’', "'").replace(u'”', '"')
|
||
m = _DIM_RE.match(t)
|
||
if not m:
|
||
return None
|
||
ft, inch, num, den = m.groups()
|
||
if ft is None and inch is None and num is None:
|
||
return None
|
||
total = 0.0
|
||
if ft:
|
||
total += float(ft) * 12.0
|
||
if inch:
|
||
total += float(inch)
|
||
if num and den and float(den):
|
||
total += float(num) / float(den)
|
||
return total
|
||
|
||
|
||
def format_inches(v):
|
||
"""Inches -> a feet-and-inches string, for reporting derived totals only."""
|
||
if v is None:
|
||
return '?'
|
||
neg = v < 0
|
||
v = abs(v)
|
||
ft = int(v // 12)
|
||
rem = v - ft * 12
|
||
inch = int(rem)
|
||
frac = rem - inch
|
||
sixteenths = int(round(frac * 16))
|
||
if sixteenths == 16:
|
||
inch += 1
|
||
sixteenths = 0
|
||
out = ''
|
||
if ft:
|
||
out += "%d'-" % ft
|
||
out += '%d' % inch
|
||
if sixteenths:
|
||
n, d = sixteenths, 16
|
||
while n % 2 == 0:
|
||
n //= 2
|
||
d //= 2
|
||
out += ' %d/%d' % (n, d)
|
||
out += '"'
|
||
return ('-' if neg else '') + out
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# raw span reading
|
||
# --------------------------------------------------------------------------
|
||
|
||
def read_spans(path):
|
||
"""Every text span in the PDF as dicts, deduplicated.
|
||
|
||
Deduplication is not tidying. This set contains 345 spans that are drawn
|
||
twice at the same point -- a dimension double-placed in the source view.
|
||
Left in, a ticket that happens to double-place a different dimension in
|
||
the new issue reports a phantom "added" string.
|
||
"""
|
||
import pymupdf
|
||
doc = pymupdf.open(path)
|
||
spans = []
|
||
seen = set()
|
||
for pno, page in enumerate(doc):
|
||
for block in page.get_text('dict')['blocks']:
|
||
if block.get('type') != 0:
|
||
continue
|
||
for line in block['lines']:
|
||
d = line.get('dir', (1.0, 0.0))
|
||
vertical = abs(d[1]) > abs(d[0])
|
||
for sp in line['spans']:
|
||
text = sp['text'].strip()
|
||
if not text:
|
||
continue
|
||
bb = sp['bbox']
|
||
key = (pno, text, round(bb[0], 1), round(bb[1], 1))
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
spans.append({
|
||
'page': pno,
|
||
'text': text,
|
||
'x': bb[0], 'y': bb[1],
|
||
'x1': bb[2], 'y1': bb[3],
|
||
'size': round(sp['size'], 1),
|
||
'font': sp['font'],
|
||
'vertical': vertical,
|
||
'block': id(block),
|
||
})
|
||
npages = doc.page_count
|
||
doc.close()
|
||
return spans, npages
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# hazards -- the four measured text-layer failure modes
|
||
# --------------------------------------------------------------------------
|
||
|
||
_COLLAPSED_RE = re.compile(r'^\d{3,}"$')
|
||
|
||
# Sheet furniture: printed on every ticket, meaning nothing about the piece,
|
||
# and grouped into whatever text block happens to be near it. Left in, it
|
||
# makes a note look reworded because the page number moved.
|
||
SHEET_FURNITURE_RE = re.compile(r'^PAGE\s+\d+\s+OF\s+\d+$', re.I)
|
||
|
||
|
||
def hazards(spans, npages):
|
||
"""Text-layer conditions that make a value untrustworthy.
|
||
|
||
Returns a list of dicts {mode, detail, page, x, y, texts}. An empty list is
|
||
a measurement, not an assumption: it means all four scans ran and found
|
||
nothing on this file.
|
||
"""
|
||
found = []
|
||
|
||
# Mode 1 -- a stacked fraction that lost its slash reads as one long
|
||
# integer. 13/16" and 1 3/16" both collapse to 1316".
|
||
for s in spans:
|
||
if _COLLAPSED_RE.match(s['text']):
|
||
found.append({'mode': 'collapsed-fraction', 'page': s['page'],
|
||
'x': s['x'], 'y': s['y'], 'texts': [s['text']],
|
||
'detail': 'reads as a single integer; may be a '
|
||
'stacked fraction that lost its slash'})
|
||
|
||
# Mode 2 -- a cipher-shifted subset font puts digits on control characters.
|
||
for s in spans:
|
||
bad = [c for c in s['text'] if ord(c) < 32 or ord(c) > 126]
|
||
if bad:
|
||
found.append({'mode': 'cipher-font', 'page': s['page'],
|
||
'x': s['x'], 'y': s['y'], 'texts': [s['text']],
|
||
'detail': 'non-printable or non-ASCII characters in '
|
||
'font %s' % s['font']})
|
||
|
||
# Mode 3 -- a page whose text layer is far thinner than its siblings has
|
||
# probably been flattened to vectors, and its stamps and callouts will be
|
||
# invisible to any text-based diff. Judged per file against its own pages,
|
||
# since a two-page ticket's pages carry comparable amounts of text.
|
||
per_page = {}
|
||
for s in spans:
|
||
per_page[s['page']] = per_page.get(s['page'], 0) + 1
|
||
for p in range(npages):
|
||
n = per_page.get(p, 0)
|
||
if n < 20:
|
||
found.append({'mode': 'thin-text-layer', 'page': p,
|
||
'x': None, 'y': None, 'texts': [],
|
||
'detail': 'only %d text spans on this page; the page '
|
||
'may be flattened to vectors, so anything '
|
||
'not listed here is not "unchanged", it is '
|
||
'unread' % n})
|
||
|
||
# Mode 4 -- retired text underneath current text. Colocated spans with
|
||
# DIFFERENT content mean get_text() is returning at least one string that
|
||
# is not what the page shows.
|
||
cells = {}
|
||
for s in spans:
|
||
k = (s['page'], round(s['x'] / 2.0), round(s['y'] / 2.0))
|
||
cells.setdefault(k, []).append(s['text'])
|
||
for k, v in sorted(cells.items()):
|
||
if len(set(v)) > 1:
|
||
found.append({'mode': 'overlaid-text', 'page': k[0],
|
||
'x': k[1] * 2.0, 'y': k[2] * 2.0, 'texts': sorted(set(v)),
|
||
'detail': 'two different strings drawn at the same '
|
||
'point; one of them is not what the sheet '
|
||
'shows and the text layer does not say which'})
|
||
return found
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# region 1 -- title block
|
||
# --------------------------------------------------------------------------
|
||
|
||
# Anchored on the printed labels, not on coordinates. A label sits above-left
|
||
# of its value in a cell; the cell's right edge is the next label on the row.
|
||
TITLE_LABELS = [
|
||
'MARK #:', 'PROJECT NAME:', 'PROJECT #:', 'SEQUENCE #', 'LENGTH:',
|
||
'WIDTH:', 'QUANITY:', 'QUANTITY:', "CONCRETE (f'c):", "CONCRETE (f'ci):",
|
||
'THICKNESS:', 'WEIGHT (lbs.):', "ADD'L PRODUCTION:", "ADD'L FINISH:",
|
||
'PRODUCT CODE:', 'FACE MIX #', 'FACE MIX (C.Y.)', 'BASE MIX#',
|
||
'BASE MIX (C.Y.)', 'DRAWN BY:', 'ENG BY:',
|
||
]
|
||
|
||
# What a field's value is allowed to look like. A value that fails its own
|
||
# pattern is reported as unreadable rather than diffed -- a title block cell
|
||
# read from the wrong side of a boundary produces a plausible wrong string,
|
||
# and a diff of two plausible wrong strings is the worst output this tool
|
||
# could emit.
|
||
_FIELD_SHAPE = {
|
||
'LENGTH:': re.compile(r'^\d+\'[-\s]*\d+(\s\d+/\d+)?"$'),
|
||
'WIDTH:': re.compile(r'^\d+\'[-\s]*\d+(\s\d+/\d+)?"$'),
|
||
'THICKNESS:': re.compile(r'^\d+(\s\d+/\d+)?"$'),
|
||
'WEIGHT (lbs.):': re.compile(r'^[\d,]+$'),
|
||
'QUANITY:': re.compile(r'^\d+$'),
|
||
'QUANTITY:': re.compile(r'^\d+$'),
|
||
"CONCRETE (f'c):": re.compile(r'^\d+$'),
|
||
"CONCRETE (f'ci):": re.compile(r'^\d+$'),
|
||
'FACE MIX (C.Y.)': re.compile(r'^[\d.]+\s*CY$'),
|
||
'BASE MIX (C.Y.)': re.compile(r'^[\d.]+\s*CY$'),
|
||
}
|
||
|
||
_ROW_TOL = 1.5
|
||
_ROW_PITCH = 11.5
|
||
|
||
|
||
def title_block(spans, claim=None):
|
||
"""{label: {'value':..., 'ok':bool, 'x':, 'y':, 'page':}} keyed by label.
|
||
|
||
`claim`, if given, is a set that collects the id() of every span consumed
|
||
here, so the leftover-callout pass does not report title-block values a
|
||
second time under a different name.
|
||
"""
|
||
all_labels = [s for s in spans if s['text'] in TITLE_LABELS]
|
||
if not all_labels:
|
||
return {}
|
||
# Every page of a ticket carries the title block. Only the first page's
|
||
# values are reported as fields -- they are the same values -- but the
|
||
# LATER pages' cells must still be claimed, or the project name reappears
|
||
# on page 2 as a sheet callout that was "added to the drawing".
|
||
pages = sorted(set(s['page'] for s in all_labels))
|
||
out = {}
|
||
for page in pages:
|
||
_title_block_page(spans, all_labels, page, out if page == pages[0] else {},
|
||
claim)
|
||
_control_numbers(spans, out, claim)
|
||
return out
|
||
|
||
|
||
def _title_block_page(spans, all_labels, page, out, claim):
|
||
labels = [s for s in all_labels if s['page'] == page]
|
||
others = [s for s in spans
|
||
if s['page'] == page and s['text'] not in TITLE_LABELS]
|
||
|
||
rows = {}
|
||
for s in labels:
|
||
for ry in rows:
|
||
if abs(ry - s['y']) <= _ROW_TOL:
|
||
rows[ry].append(s)
|
||
break
|
||
else:
|
||
rows[s['y']] = [s]
|
||
row_ys = sorted(rows)
|
||
|
||
out = {}
|
||
for i, ry in enumerate(row_ys):
|
||
cells = sorted(rows[ry], key=lambda s: s['x'])
|
||
nxt_row = row_ys[i + 1] if i + 1 < len(row_ys) else None
|
||
y_hi = ry + _ROW_PITCH
|
||
if nxt_row is not None:
|
||
y_hi = min(y_hi, nxt_row - _ROW_TOL)
|
||
for j, lab in enumerate(cells):
|
||
x_lo = lab['x'] - 5.0
|
||
x_hi = cells[j + 1]['x'] - 1.0 if j + 1 < len(cells) else 1e9
|
||
vals = [s for s in others
|
||
if x_lo <= s['x'] < x_hi and ry - _ROW_TOL <= s['y'] <= y_hi]
|
||
vals.sort(key=lambda s: (s['y'], s['x']))
|
||
text = ' '.join(v['text'] for v in vals).strip()
|
||
shape = _FIELD_SHAPE.get(lab['text'])
|
||
ok = True
|
||
if text and shape and not shape.match(text):
|
||
ok = False
|
||
out[lab['text']] = {'value': text, 'ok': ok, 'page': page,
|
||
'x': lab['x'], 'y': lab['y']}
|
||
if claim is not None:
|
||
claim.add(id(lab))
|
||
for v in vals:
|
||
claim.add(id(v))
|
||
|
||
return out
|
||
|
||
|
||
def _control_numbers(spans, out, claim):
|
||
# Control numbers are the ticket's identity under this plant's rule (one
|
||
# ticket per control number, not per mark), and they are printed outside
|
||
# the title-block grid.
|
||
for s in spans:
|
||
if s['text'].upper().startswith('CONTROL NUMBER'):
|
||
near = [o for o in spans
|
||
if o['page'] == s['page'] and o['x'] > s['x']
|
||
and abs(o['y'] - s['y']) < 12 and re.match(r'^[\d,\s]+$', o['text'])]
|
||
near.sort(key=lambda o: o['x'])
|
||
out['CONTROL NUMBERS:'] = {
|
||
'value': ' '.join(o['text'] for o in near).strip(),
|
||
'ok': True, 'page': s['page'], 'x': s['x'], 'y': s['y']}
|
||
if claim is not None:
|
||
claim.add(id(s))
|
||
for o in near:
|
||
claim.add(id(o))
|
||
break
|
||
return out
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# region 2 -- bill of material
|
||
# --------------------------------------------------------------------------
|
||
|
||
def bom(spans, claim=None):
|
||
"""[{'qty','mark','desc','page','x','y'}] from the QTY/MARK/DESCRIPTION table.
|
||
|
||
Anchored on the header row. Description continuation lines carry no qty and
|
||
no mark, so rows are opened by a qty cell and closed by the next one.
|
||
"""
|
||
hdr_q = hdr_m = hdr_d = None
|
||
for s in spans:
|
||
if s['text'] == 'QTY':
|
||
cand_m = [o for o in spans if o['page'] == s['page']
|
||
and o['text'] == 'MARK' and abs(o['y'] - s['y']) < 2
|
||
and o['x'] > s['x']]
|
||
cand_d = [o for o in spans if o['page'] == s['page']
|
||
and o['text'] == 'DESCRIPTION' and abs(o['y'] - s['y']) < 2
|
||
and o['x'] > s['x']]
|
||
if cand_m:
|
||
hdr_q, hdr_m = s, cand_m[0]
|
||
hdr_d = cand_d[0] if cand_d else None
|
||
break
|
||
if hdr_q is None:
|
||
return []
|
||
|
||
page = hdr_q['page']
|
||
x_mark = hdr_m['x'] - 8.0
|
||
x_desc = (hdr_d['x'] - 4.0) if hdr_d else (x_mark + 40.0)
|
||
# The mark column is narrow. Widening it to "everything before the
|
||
# DESCRIPTION column" was measured to swallow a handling-diagram label
|
||
# sitting at x=1055 and report the strand row's mark as ERECTION -- a
|
||
# phantom "item added / item removed" pair on a ticket whose bill did not
|
||
# change at all.
|
||
x_mark_hi = min(x_desc, hdr_m['x'] + 35.0)
|
||
body = [s for s in spans if s['page'] == page and s['y'] > hdr_q['y'] + 2
|
||
and s['x'] >= hdr_q['x'] - 8.0
|
||
and s['text'] not in HANDLING_LABELS]
|
||
|
||
# A row is opened by a bare integer in the QTY column. Requiring the
|
||
# integer matters: a note block printed below the table sits in the same
|
||
# x-band, and without this it opens phantom rows and swallows the strand
|
||
# row's description.
|
||
qtys = sorted([s for s in body if s['x'] < x_mark and re.match(r'^\d+$', s['text'])],
|
||
key=lambda s: s['y'])
|
||
if not qtys:
|
||
return []
|
||
marks = sorted([s for s in body if x_mark <= s['x'] < x_mark_hi],
|
||
key=lambda s: (s['y'], s['x']))
|
||
descs = [s for s in body if s['x'] >= x_desc]
|
||
|
||
# Row pitch, measured on this ticket rather than assumed, bounds the last
|
||
# row so it cannot absorb whatever is printed under the table.
|
||
gaps = sorted(qtys[i + 1]['y'] - qtys[i]['y'] for i in range(len(qtys) - 1))
|
||
pitch = gaps[len(gaps) // 2] if gaps else 11.0
|
||
|
||
if claim is not None:
|
||
claim.add(id(hdr_q))
|
||
claim.add(id(hdr_m))
|
||
if hdr_d is not None:
|
||
claim.add(id(hdr_d))
|
||
|
||
rows = []
|
||
bbox = [hdr_q['x'] - 10.0, hdr_q['y'] - 4.0,
|
||
max(sp['x1'] for sp in body) if body else hdr_q['x'] + 250.0,
|
||
max(q['y'] for q in qtys) + pitch * 1.8]
|
||
for i, q in enumerate(qtys):
|
||
y_hi = q['y'] + pitch * 1.8
|
||
if i + 1 < len(qtys):
|
||
y_hi = min(y_hi, qtys[i + 1]['y'] - 2.0)
|
||
mk = [m for m in marks if q['y'] - 2.0 <= m['y'] < y_hi]
|
||
ds = sorted([d for d in descs if q['y'] - 2.0 <= d['y'] < y_hi],
|
||
key=lambda s: (s['y'], s['x']))
|
||
rows.append({
|
||
'qty': q['text'],
|
||
'mark': mk[0]['text'] if mk else '',
|
||
'desc': ' '.join(d['text'] for d in ds),
|
||
'page': page, 'x': q['x'], 'y': q['y'],
|
||
})
|
||
if claim is not None:
|
||
claim.add(id(q))
|
||
for s in mk + ds:
|
||
claim.add(id(s))
|
||
if claim is not None:
|
||
claim.add(('bom_bbox', page) + tuple(bbox))
|
||
return rows
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# region 3 -- handling diagrams
|
||
# --------------------------------------------------------------------------
|
||
|
||
HANDLING_LABELS = ('DUNNAGE', 'PICKING/STRIPPING', 'PICKING', 'STRIPPING',
|
||
'ERECTION', 'SHIPPING', 'TRANSPORTATION')
|
||
|
||
|
||
# A handling dimension is printed a fixed distance BELOW its diagram's label
|
||
# and starting at the label's own x. Both numbers are measurements, not
|
||
# guesses: across all 26 tickets in the two issues the offset is 43-46 points
|
||
# and never anything else, and the rule reproduces every diagram on every file.
|
||
#
|
||
# The rule this replaced -- "everything in the label's column, between this
|
||
# label and the next one down" -- was wrong because the diagrams are not always
|
||
# in one column. Half the set stacks all three at one x; the other half puts
|
||
# PICKING/STRIPPING at x=910 and ERECTION and DUNNAGE side by side at x=1055.
|
||
# WP194, WP195 and WP200 CHANGE LAYOUT between the two issues, so a
|
||
# column-based reader compared a four-dimension picking diagram against a
|
||
# two-dimension one and called it a material change on three tickets where
|
||
# nothing had moved.
|
||
_HANDLING_DY_MIN = 33.0
|
||
_HANDLING_DY_MAX = 62.0
|
||
_HANDLING_DX_MIN = -18.0
|
||
_HANDLING_DX_MAX = 210.0
|
||
|
||
|
||
def handling(spans):
|
||
"""{diagram label: [dim spans left-to-right]} for the little stage diagrams.
|
||
|
||
Order is PRINTED order, left to right. It is not mirrored and it is not
|
||
sorted by value. That matters more than it looks: reading a pair of dunnage
|
||
insets the other way round was measured, on this very ticket set, to turn
|
||
an engineer's CORRECTED sheet back into a failure. A diff that disagrees
|
||
with a fix the reader has already made is a diff he stops opening.
|
||
"""
|
||
labels = [s for s in spans if s['text'] in HANDLING_LABELS]
|
||
out = {}
|
||
for lab in sorted(labels, key=lambda s: (s['page'], s['y'], s['x'])):
|
||
dims = [s for s in spans
|
||
if s['page'] == lab['page']
|
||
and lab['y'] + _HANDLING_DY_MIN <= s['y'] <= lab['y'] + _HANDLING_DY_MAX
|
||
and lab['x'] + _HANDLING_DX_MIN <= s['x'] <= lab['x'] + _HANDLING_DX_MAX
|
||
and parse_dim(s['text']) is not None]
|
||
dims.sort(key=lambda s: s['x'])
|
||
key = lab['text']
|
||
n = 2
|
||
while key in out:
|
||
key = '%s (%d)' % (lab['text'], n)
|
||
n += 1
|
||
out[key] = dims
|
||
return out
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# region 4 -- note blocks
|
||
# --------------------------------------------------------------------------
|
||
|
||
_NOTE_HEAD_RE = re.compile(r'^(PRODUCTION\s+)?NOTES?\s*:?$', re.I)
|
||
|
||
|
||
def notes(spans, exclude=None):
|
||
"""[{'text','page','x','y'}] -- each note, from its heading down.
|
||
|
||
A note is its HEADING plus the lines under it, not "whatever pymupdf put
|
||
in the same block". Measured reason: on page 2 of one ticket the block
|
||
containing PRODUCTION NOTE also contains a 6" dimension and an SR567 bar
|
||
tag drawn nearby, and taking the block whole reported the note as removed
|
||
on one issue and added on the other because a bar tag next to it had
|
||
moved. Anchoring on the heading makes the note the paragraph an engineer
|
||
would read out, so an added or reworded note surfaces once, as a
|
||
paragraph, rather than as four orphan lines.
|
||
"""
|
||
by_block = {}
|
||
for s in spans:
|
||
by_block.setdefault(s['block'], []).append(s)
|
||
out = []
|
||
for _, ss in by_block.items():
|
||
keep = [x for x in sorted(ss, key=lambda x: (x['y'], x['x']))
|
||
if x['text'] not in TITLE_LABELS
|
||
and not SHEET_FURNITURE_RE.match(x['text'])
|
||
and not (exclude and id(x) in exclude)]
|
||
heads = [i for i, x in enumerate(keep) if _NOTE_HEAD_RE.match(x['text'])]
|
||
for n, i in enumerate(heads):
|
||
j = heads[n + 1] if n + 1 < len(heads) else len(keep)
|
||
body = keep[i:j]
|
||
out.append({'text': ' '.join(' '.join(x['text'] for x in body).split()),
|
||
'page': body[0]['page'], 'x': body[0]['x'],
|
||
'y': body[0]['y'],
|
||
'ids': [id(x) for x in body]})
|
||
out.sort(key=lambda n: (n['page'], n['y'], n['x']))
|
||
return out
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# region 5 -- dimension chains on the drawing
|
||
# --------------------------------------------------------------------------
|
||
|
||
_CHAIN_TOL = 2.5
|
||
|
||
|
||
def chains(spans, exclude):
|
||
"""Dimension strings grouped into the chains they are printed in.
|
||
|
||
A dimension chain is what a detailer actually edits: a run of segments
|
||
along one line, adding up to an overall. Grouping by the printed line and
|
||
comparing the run in order is what turns "two numbers differ somewhere on
|
||
a D-size sheet" into "the second and third segments of the top chain on the
|
||
bed plan swapped 3'-3"/1'-0" for 2'-9"/1'-6", total unchanged".
|
||
|
||
`exclude` is the set of span ids already claimed by another region, so a
|
||
handling-diagram dimension is not also reported as a loose chain.
|
||
"""
|
||
dims = [s for s in spans
|
||
if id(s) not in exclude and parse_dim(s['text']) is not None]
|
||
groups = []
|
||
for vert in (False, True):
|
||
pool = [s for s in dims if s['vertical'] == vert]
|
||
used = set()
|
||
for s in sorted(pool, key=lambda s: (s['page'], s['x'], s['y'])):
|
||
if id(s) in used:
|
||
continue
|
||
axis = 'x' if vert else 'y'
|
||
run = [o for o in pool if o['page'] == s['page']
|
||
and id(o) not in used and abs(o[axis] - s[axis]) <= _CHAIN_TOL]
|
||
for o in run:
|
||
used.add(id(o))
|
||
run.sort(key=lambda o: o['y'] if vert else o['x'])
|
||
other = 'y' if vert else 'x'
|
||
groups.append({
|
||
'page': s['page'],
|
||
'vertical': vert,
|
||
# `pos` is across the run (which line the chain sits on);
|
||
# `along` is where along that line it starts. Both are needed:
|
||
# two section details on the same page share an orientation and
|
||
# can share a `pos`, and matching on `pos` alone paired a
|
||
# dimension in SECTION A with one in SECTION B and called it a
|
||
# change on a ticket whose dimension text was byte-identical.
|
||
'pos': round(sum(o[axis] for o in run) / float(len(run)), 1),
|
||
'along': round(min(o[other] for o in run), 1),
|
||
'items': run,
|
||
})
|
||
groups.sort(key=lambda g: (g['page'], g['vertical'], g['pos']))
|
||
return groups
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# region 6 -- callouts and view titles
|
||
# --------------------------------------------------------------------------
|
||
|
||
_TITLE_WORDS = ('PLAN', 'SECTION', 'ELEVATION', 'DETAIL', 'VIEW', 'SCHEDULE')
|
||
|
||
|
||
def view_titles(spans):
|
||
"""Large-font strings that name a view, used as human-readable locators.
|
||
|
||
The size test alone is not enough: a section bubble's "A", the "MK" tag on
|
||
the bed plan and the scale string "1/4" = 1'-0"" are all set at the title
|
||
size, and naming a finding's location "MK, page 1" tells a reader nothing.
|
||
A view title on these sheets always contains one of the drawing words.
|
||
"""
|
||
out = []
|
||
for s in spans:
|
||
if s['size'] < 12.0:
|
||
continue
|
||
t = s['text'].upper()
|
||
if any(w in t for w in _TITLE_WORDS):
|
||
out.append(s)
|
||
return out
|
||
|
||
|
||
def locate(spans_titles, page, x, y, page_rect=(1224.0, 792.0)):
|
||
"""Name a point on the sheet the way somebody hunting for it would.
|
||
|
||
View title plus a coarse zone. "BED PLAN, page 1" on a D-size sheet is
|
||
still a lot of paper to search; "BED PLAN, page 1, lower left" is not.
|
||
"""
|
||
w, h = page_rect
|
||
col = 'left' if x < w / 3.0 else ('centre' if x < 2 * w / 3.0 else 'right')
|
||
row = 'upper' if y < h / 3.0 else ('middle' if y < 2 * h / 3.0 else 'lower')
|
||
zone = '%s %s' % (row, col)
|
||
cands = [t for t in spans_titles if t['page'] == page]
|
||
if not cands:
|
||
return 'page %d, %s' % (page + 1, zone)
|
||
best = min(cands, key=lambda t: (t['x'] - x) ** 2 + (t['y'] - y) ** 2)
|
||
return '%s, page %d, %s' % (best['text'], page + 1, zone)
|
||
|
||
|
||
def callouts(spans, claimed):
|
||
"""Everything left over that is a real annotation rather than a dimension.
|
||
|
||
Deliberately excludes anything already claimed by a region and anything
|
||
that parses as a dimension. What is left is part callouts (P76 W/ SR5602
|
||
(S.I.F.)), bar tags, spacing strings, section bubbles.
|
||
"""
|
||
out = []
|
||
for s in spans:
|
||
if id(s) in claimed:
|
||
continue
|
||
if parse_dim(s['text']) is not None:
|
||
continue
|
||
if len(s['text']) < 2:
|
||
continue
|
||
out.append(s)
|
||
return out
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# top level
|
||
# --------------------------------------------------------------------------
|
||
|
||
_TS_RE = re.compile(r'^\d{1,2}/\d{1,2}/\d{2,4}(\s+\d{1,2}:\d{2}(:\d{2})?\s*(AM|PM)?)?$',
|
||
re.I)
|
||
|
||
|
||
def extract(path):
|
||
"""One ticket PDF -> the structure ticketdelta.compare() consumes."""
|
||
spans, npages = read_spans(path)
|
||
haz = hazards(spans, npages)
|
||
|
||
claimed = set()
|
||
tb = title_block(spans, claimed)
|
||
bm = bom(spans, claimed)
|
||
bom_box = None
|
||
for c in claimed:
|
||
if isinstance(c, tuple) and c and c[0] == 'bom_bbox':
|
||
bom_box = {'page': c[1], 'x0': c[2], 'y0': c[3],
|
||
'x1': c[4], 'y1': c[5]}
|
||
claimed = set(c for c in claimed if not isinstance(c, tuple))
|
||
hl = handling(spans)
|
||
nt = notes(spans, claimed)
|
||
titles = view_titles(spans)
|
||
|
||
for _, dims in hl.items():
|
||
for d in dims:
|
||
claimed.add(id(d))
|
||
for s in spans:
|
||
if s['text'] in HANDLING_LABELS:
|
||
claimed.add(id(s))
|
||
# Note blocks are compared as paragraphs, so their lines must not also
|
||
# surface as loose callouts.
|
||
for n in nt:
|
||
for i in n['ids']:
|
||
claimed.add(i)
|
||
|
||
ch = chains(spans, claimed)
|
||
for g in ch:
|
||
for it in g['items']:
|
||
claimed.add(id(it))
|
||
|
||
stamps = [s['text'] for s in spans if _TS_RE.match(s['text'])]
|
||
|
||
return {
|
||
'path': path,
|
||
'file': os.path.basename(path),
|
||
'pages': npages,
|
||
'hazards': haz,
|
||
'title_block': tb,
|
||
'bom': bm,
|
||
'bom_bbox': bom_box,
|
||
'handling': hl,
|
||
'notes': nt,
|
||
'chains': ch,
|
||
'view_titles': titles,
|
||
'callouts': callouts(spans, claimed),
|
||
'timestamps': sorted(set(stamps)),
|
||
'span_count': len(spans),
|
||
}
|