# -*- coding: utf-8 -*- """Read one shop-ticket PDF into a structure a revision diff can reason about. WHY THIS MODULE EXISTS A ticket is not a page of prose. It is a title block, a bill of material, three handling diagrams, a set of note blocks, and a drawing carrying dimension chains and part callouts. Those are the things a precast engineer names when he says what changed. A diff that works on lines of text says "line 47 differs"; a diff that works on these says "the dunnage diagram's near-end inset went from 4'-9" to 5'-0"". Only the second one gets read. So extraction is region-aware, and every region is found by an ANCHOR, never by an absolute coordinate. Measured reason: WP199 puts its handling diagrams at x=819 and WP219 puts them at x=910. A coordinate-keyed extractor would have silently returned nothing for half the set and reported "no changes". WHAT THIS MODULE REFUSES TO DO It does not guess. Four PDF text-layer failure modes were measured on this project's other source documents, and each one produces a CONFIDENT WRONG NUMBER rather than an error: 1. stacked fractions collapsing ("1316"" for both 13/16" and 1 3/16") 2. cipher-shifted subset fonts (characters mapped 29 code points down) 3. stamps invisible to text search 4. retired text sitting underneath current text, returned first A diff is uniquely exposed to these, because a misread on ONE SIDE ONLY invents a change that never happened. So `hazards()` scans for all four and the diff downgrades or drops anything they touch. Measured on this ticket set: modes 1 and 2 are absent (fractions come through as "1/2", no non-ASCII byte in 46,079 spans), mode 4 is present (345 exactly-colocated duplicate spans, 2 colocated pairs with DIFFERENT text). That measurement is what licenses reading these particular numbers off the text layer; it is not a general licence, which is why the scan runs on every file every time. CPython only -- pymupdf is not available to IronPython, so the Revit-side code must not import this. The comparison half lives in ticketdelta.py and is pure. """ import os import re # -------------------------------------------------------------------------- # dimension parsing # -------------------------------------------------------------------------- # A ticket writes the same length two ways depending on which family drew it: # the bed plan prints 1' - 0", the handling diagrams print 1'-0". Comparing the # strings would report a change on every ticket. Compare the VALUE, show the # RAW TEXT. _DIM_RE = re.compile( r'^(?:(\d+)\'\s*-?\s*)?' # feet r'(?:(\d+)(?!/))?' # whole inches (not the numerator of a fraction) r'\s*(?:(\d+)/(\d+))?' # fractional inches r'"$' ) def parse_dim(text): """Feet-and-inches string -> inches as a float, or None if not a dimension. Accepts 22'-11", 22' - 11", 2' - 5 1/2", 11 1/2", 8", 3/4". Returns None for anything else -- including things that merely contain a dimension, such as (5) @ 1'-6" = 7'-6", which is a note, not a dimension. """ if not text: return None t = text.strip().replace(u'’', "'").replace(u'”', '"') m = _DIM_RE.match(t) if not m: return None ft, inch, num, den = m.groups() if ft is None and inch is None and num is None: return None total = 0.0 if ft: total += float(ft) * 12.0 if inch: total += float(inch) if num and den and float(den): total += float(num) / float(den) return total def format_inches(v): """Inches -> a feet-and-inches string, for reporting derived totals only.""" if v is None: return '?' neg = v < 0 v = abs(v) ft = int(v // 12) rem = v - ft * 12 inch = int(rem) frac = rem - inch sixteenths = int(round(frac * 16)) if sixteenths == 16: inch += 1 sixteenths = 0 out = '' if ft: out += "%d'-" % ft out += '%d' % inch if sixteenths: n, d = sixteenths, 16 while n % 2 == 0: n //= 2 d //= 2 out += ' %d/%d' % (n, d) out += '"' return ('-' if neg else '') + out # -------------------------------------------------------------------------- # raw span reading # -------------------------------------------------------------------------- def read_spans(path): """Every text span in the PDF as dicts, deduplicated. Deduplication is not tidying. This set contains 345 spans that are drawn twice at the same point -- a dimension double-placed in the source view. Left in, a ticket that happens to double-place a different dimension in the new issue reports a phantom "added" string. """ import pymupdf doc = pymupdf.open(path) spans = [] seen = set() for pno, page in enumerate(doc): for block in page.get_text('dict')['blocks']: if block.get('type') != 0: continue for line in block['lines']: d = line.get('dir', (1.0, 0.0)) vertical = abs(d[1]) > abs(d[0]) for sp in line['spans']: text = sp['text'].strip() if not text: continue bb = sp['bbox'] key = (pno, text, round(bb[0], 1), round(bb[1], 1)) if key in seen: continue seen.add(key) spans.append({ 'page': pno, 'text': text, 'x': bb[0], 'y': bb[1], 'x1': bb[2], 'y1': bb[3], 'size': round(sp['size'], 1), 'font': sp['font'], 'vertical': vertical, 'block': id(block), }) npages = doc.page_count doc.close() return spans, npages # -------------------------------------------------------------------------- # hazards -- the four measured text-layer failure modes # -------------------------------------------------------------------------- _COLLAPSED_RE = re.compile(r'^\d{3,}"$') # Sheet furniture: printed on every ticket, meaning nothing about the piece, # and grouped into whatever text block happens to be near it. Left in, it # makes a note look reworded because the page number moved. SHEET_FURNITURE_RE = re.compile(r'^PAGE\s+\d+\s+OF\s+\d+$', re.I) def hazards(spans, npages): """Text-layer conditions that make a value untrustworthy. Returns a list of dicts {mode, detail, page, x, y, texts}. An empty list is a measurement, not an assumption: it means all four scans ran and found nothing on this file. """ found = [] # Mode 1 -- a stacked fraction that lost its slash reads as one long # integer. 13/16" and 1 3/16" both collapse to 1316". for s in spans: if _COLLAPSED_RE.match(s['text']): found.append({'mode': 'collapsed-fraction', 'page': s['page'], 'x': s['x'], 'y': s['y'], 'texts': [s['text']], 'detail': 'reads as a single integer; may be a ' 'stacked fraction that lost its slash'}) # Mode 2 -- a cipher-shifted subset font puts digits on control characters. for s in spans: bad = [c for c in s['text'] if ord(c) < 32 or ord(c) > 126] if bad: found.append({'mode': 'cipher-font', 'page': s['page'], 'x': s['x'], 'y': s['y'], 'texts': [s['text']], 'detail': 'non-printable or non-ASCII characters in ' 'font %s' % s['font']}) # Mode 3 -- a page whose text layer is far thinner than its siblings has # probably been flattened to vectors, and its stamps and callouts will be # invisible to any text-based diff. Judged per file against its own pages, # since a two-page ticket's pages carry comparable amounts of text. per_page = {} for s in spans: per_page[s['page']] = per_page.get(s['page'], 0) + 1 for p in range(npages): n = per_page.get(p, 0) if n < 20: found.append({'mode': 'thin-text-layer', 'page': p, 'x': None, 'y': None, 'texts': [], 'detail': 'only %d text spans on this page; the page ' 'may be flattened to vectors, so anything ' 'not listed here is not "unchanged", it is ' 'unread' % n}) # Mode 4 -- retired text underneath current text. Colocated spans with # DIFFERENT content mean get_text() is returning at least one string that # is not what the page shows. cells = {} for s in spans: k = (s['page'], round(s['x'] / 2.0), round(s['y'] / 2.0)) cells.setdefault(k, []).append(s['text']) for k, v in sorted(cells.items()): if len(set(v)) > 1: found.append({'mode': 'overlaid-text', 'page': k[0], 'x': k[1] * 2.0, 'y': k[2] * 2.0, 'texts': sorted(set(v)), 'detail': 'two different strings drawn at the same ' 'point; one of them is not what the sheet ' 'shows and the text layer does not say which'}) return found # -------------------------------------------------------------------------- # region 1 -- title block # -------------------------------------------------------------------------- # Anchored on the printed labels, not on coordinates. A label sits above-left # of its value in a cell; the cell's right edge is the next label on the row. TITLE_LABELS = [ 'MARK #:', 'PROJECT NAME:', 'PROJECT #:', 'SEQUENCE #', 'LENGTH:', 'WIDTH:', 'QUANITY:', 'QUANTITY:', "CONCRETE (f'c):", "CONCRETE (f'ci):", 'THICKNESS:', 'WEIGHT (lbs.):', "ADD'L PRODUCTION:", "ADD'L FINISH:", 'PRODUCT CODE:', 'FACE MIX #', 'FACE MIX (C.Y.)', 'BASE MIX#', 'BASE MIX (C.Y.)', 'DRAWN BY:', 'ENG BY:', ] # What a field's value is allowed to look like. A value that fails its own # pattern is reported as unreadable rather than diffed -- a title block cell # read from the wrong side of a boundary produces a plausible wrong string, # and a diff of two plausible wrong strings is the worst output this tool # could emit. _FIELD_SHAPE = { 'LENGTH:': re.compile(r'^\d+\'[-\s]*\d+(\s\d+/\d+)?"$'), 'WIDTH:': re.compile(r'^\d+\'[-\s]*\d+(\s\d+/\d+)?"$'), 'THICKNESS:': re.compile(r'^\d+(\s\d+/\d+)?"$'), 'WEIGHT (lbs.):': re.compile(r'^[\d,]+$'), 'QUANITY:': re.compile(r'^\d+$'), 'QUANTITY:': re.compile(r'^\d+$'), "CONCRETE (f'c):": re.compile(r'^\d+$'), "CONCRETE (f'ci):": re.compile(r'^\d+$'), 'FACE MIX (C.Y.)': re.compile(r'^[\d.]+\s*CY$'), 'BASE MIX (C.Y.)': re.compile(r'^[\d.]+\s*CY$'), } _ROW_TOL = 1.5 _ROW_PITCH = 11.5 def title_block(spans, claim=None): """{label: {'value':..., 'ok':bool, 'x':, 'y':, 'page':}} keyed by label. `claim`, if given, is a set that collects the id() of every span consumed here, so the leftover-callout pass does not report title-block values a second time under a different name. """ all_labels = [s for s in spans if s['text'] in TITLE_LABELS] if not all_labels: return {} # Every page of a ticket carries the title block. Only the first page's # values are reported as fields -- they are the same values -- but the # LATER pages' cells must still be claimed, or the project name reappears # on page 2 as a sheet callout that was "added to the drawing". pages = sorted(set(s['page'] for s in all_labels)) out = {} for page in pages: _title_block_page(spans, all_labels, page, out if page == pages[0] else {}, claim) _control_numbers(spans, out, claim) return out def _title_block_page(spans, all_labels, page, out, claim): labels = [s for s in all_labels if s['page'] == page] others = [s for s in spans if s['page'] == page and s['text'] not in TITLE_LABELS] rows = {} for s in labels: for ry in rows: if abs(ry - s['y']) <= _ROW_TOL: rows[ry].append(s) break else: rows[s['y']] = [s] row_ys = sorted(rows) out = {} for i, ry in enumerate(row_ys): cells = sorted(rows[ry], key=lambda s: s['x']) nxt_row = row_ys[i + 1] if i + 1 < len(row_ys) else None y_hi = ry + _ROW_PITCH if nxt_row is not None: y_hi = min(y_hi, nxt_row - _ROW_TOL) for j, lab in enumerate(cells): x_lo = lab['x'] - 5.0 x_hi = cells[j + 1]['x'] - 1.0 if j + 1 < len(cells) else 1e9 vals = [s for s in others if x_lo <= s['x'] < x_hi and ry - _ROW_TOL <= s['y'] <= y_hi] vals.sort(key=lambda s: (s['y'], s['x'])) text = ' '.join(v['text'] for v in vals).strip() shape = _FIELD_SHAPE.get(lab['text']) ok = True if text and shape and not shape.match(text): ok = False out[lab['text']] = {'value': text, 'ok': ok, 'page': page, 'x': lab['x'], 'y': lab['y']} if claim is not None: claim.add(id(lab)) for v in vals: claim.add(id(v)) return out def _control_numbers(spans, out, claim): # Control numbers are the ticket's identity under this plant's rule (one # ticket per control number, not per mark), and they are printed outside # the title-block grid. for s in spans: if s['text'].upper().startswith('CONTROL NUMBER'): near = [o for o in spans if o['page'] == s['page'] and o['x'] > s['x'] and abs(o['y'] - s['y']) < 12 and re.match(r'^[\d,\s]+$', o['text'])] near.sort(key=lambda o: o['x']) out['CONTROL NUMBERS:'] = { 'value': ' '.join(o['text'] for o in near).strip(), 'ok': True, 'page': s['page'], 'x': s['x'], 'y': s['y']} if claim is not None: claim.add(id(s)) for o in near: claim.add(id(o)) break return out # -------------------------------------------------------------------------- # region 2 -- bill of material # -------------------------------------------------------------------------- def bom(spans, claim=None): """[{'qty','mark','desc','page','x','y'}] from the QTY/MARK/DESCRIPTION table. Anchored on the header row. Description continuation lines carry no qty and no mark, so rows are opened by a qty cell and closed by the next one. """ hdr_q = hdr_m = hdr_d = None for s in spans: if s['text'] == 'QTY': cand_m = [o for o in spans if o['page'] == s['page'] and o['text'] == 'MARK' and abs(o['y'] - s['y']) < 2 and o['x'] > s['x']] cand_d = [o for o in spans if o['page'] == s['page'] and o['text'] == 'DESCRIPTION' and abs(o['y'] - s['y']) < 2 and o['x'] > s['x']] if cand_m: hdr_q, hdr_m = s, cand_m[0] hdr_d = cand_d[0] if cand_d else None break if hdr_q is None: return [] page = hdr_q['page'] x_mark = hdr_m['x'] - 8.0 x_desc = (hdr_d['x'] - 4.0) if hdr_d else (x_mark + 40.0) # The mark column is narrow. Widening it to "everything before the # DESCRIPTION column" was measured to swallow a handling-diagram label # sitting at x=1055 and report the strand row's mark as ERECTION -- a # phantom "item added / item removed" pair on a ticket whose bill did not # change at all. x_mark_hi = min(x_desc, hdr_m['x'] + 35.0) body = [s for s in spans if s['page'] == page and s['y'] > hdr_q['y'] + 2 and s['x'] >= hdr_q['x'] - 8.0 and s['text'] not in HANDLING_LABELS] # A row is opened by a bare integer in the QTY column. Requiring the # integer matters: a note block printed below the table sits in the same # x-band, and without this it opens phantom rows and swallows the strand # row's description. qtys = sorted([s for s in body if s['x'] < x_mark and re.match(r'^\d+$', s['text'])], key=lambda s: s['y']) if not qtys: return [] marks = sorted([s for s in body if x_mark <= s['x'] < x_mark_hi], key=lambda s: (s['y'], s['x'])) descs = [s for s in body if s['x'] >= x_desc] # Row pitch, measured on this ticket rather than assumed, bounds the last # row so it cannot absorb whatever is printed under the table. gaps = sorted(qtys[i + 1]['y'] - qtys[i]['y'] for i in range(len(qtys) - 1)) pitch = gaps[len(gaps) // 2] if gaps else 11.0 if claim is not None: claim.add(id(hdr_q)) claim.add(id(hdr_m)) if hdr_d is not None: claim.add(id(hdr_d)) rows = [] bbox = [hdr_q['x'] - 10.0, hdr_q['y'] - 4.0, max(sp['x1'] for sp in body) if body else hdr_q['x'] + 250.0, max(q['y'] for q in qtys) + pitch * 1.8] for i, q in enumerate(qtys): y_hi = q['y'] + pitch * 1.8 if i + 1 < len(qtys): y_hi = min(y_hi, qtys[i + 1]['y'] - 2.0) mk = [m for m in marks if q['y'] - 2.0 <= m['y'] < y_hi] ds = sorted([d for d in descs if q['y'] - 2.0 <= d['y'] < y_hi], key=lambda s: (s['y'], s['x'])) rows.append({ 'qty': q['text'], 'mark': mk[0]['text'] if mk else '', 'desc': ' '.join(d['text'] for d in ds), 'page': page, 'x': q['x'], 'y': q['y'], }) if claim is not None: claim.add(id(q)) for s in mk + ds: claim.add(id(s)) if claim is not None: claim.add(('bom_bbox', page) + tuple(bbox)) return rows # -------------------------------------------------------------------------- # region 3 -- handling diagrams # -------------------------------------------------------------------------- HANDLING_LABELS = ('DUNNAGE', 'PICKING/STRIPPING', 'PICKING', 'STRIPPING', 'ERECTION', 'SHIPPING', 'TRANSPORTATION') # A handling dimension is printed a fixed distance BELOW its diagram's label # and starting at the label's own x. Both numbers are measurements, not # guesses: across all 26 tickets in the two issues the offset is 43-46 points # and never anything else, and the rule reproduces every diagram on every file. # # The rule this replaced -- "everything in the label's column, between this # label and the next one down" -- was wrong because the diagrams are not always # in one column. Half the set stacks all three at one x; the other half puts # PICKING/STRIPPING at x=910 and ERECTION and DUNNAGE side by side at x=1055. # WP194, WP195 and WP200 CHANGE LAYOUT between the two issues, so a # column-based reader compared a four-dimension picking diagram against a # two-dimension one and called it a material change on three tickets where # nothing had moved. _HANDLING_DY_MIN = 33.0 _HANDLING_DY_MAX = 62.0 _HANDLING_DX_MIN = -18.0 _HANDLING_DX_MAX = 210.0 def handling(spans): """{diagram label: [dim spans left-to-right]} for the little stage diagrams. Order is PRINTED order, left to right. It is not mirrored and it is not sorted by value. That matters more than it looks: reading a pair of dunnage insets the other way round was measured, on this very ticket set, to turn an engineer's CORRECTED sheet back into a failure. A diff that disagrees with a fix the reader has already made is a diff he stops opening. """ labels = [s for s in spans if s['text'] in HANDLING_LABELS] out = {} for lab in sorted(labels, key=lambda s: (s['page'], s['y'], s['x'])): dims = [s for s in spans if s['page'] == lab['page'] and lab['y'] + _HANDLING_DY_MIN <= s['y'] <= lab['y'] + _HANDLING_DY_MAX and lab['x'] + _HANDLING_DX_MIN <= s['x'] <= lab['x'] + _HANDLING_DX_MAX and parse_dim(s['text']) is not None] dims.sort(key=lambda s: s['x']) key = lab['text'] n = 2 while key in out: key = '%s (%d)' % (lab['text'], n) n += 1 out[key] = dims return out # -------------------------------------------------------------------------- # region 4 -- note blocks # -------------------------------------------------------------------------- _NOTE_HEAD_RE = re.compile(r'^(PRODUCTION\s+)?NOTES?\s*:?$', re.I) def notes(spans, exclude=None): """[{'text','page','x','y'}] -- each note, from its heading down. A note is its HEADING plus the lines under it, not "whatever pymupdf put in the same block". Measured reason: on page 2 of one ticket the block containing PRODUCTION NOTE also contains a 6" dimension and an SR567 bar tag drawn nearby, and taking the block whole reported the note as removed on one issue and added on the other because a bar tag next to it had moved. Anchoring on the heading makes the note the paragraph an engineer would read out, so an added or reworded note surfaces once, as a paragraph, rather than as four orphan lines. """ by_block = {} for s in spans: by_block.setdefault(s['block'], []).append(s) out = [] for _, ss in by_block.items(): keep = [x for x in sorted(ss, key=lambda x: (x['y'], x['x'])) if x['text'] not in TITLE_LABELS and not SHEET_FURNITURE_RE.match(x['text']) and not (exclude and id(x) in exclude)] heads = [i for i, x in enumerate(keep) if _NOTE_HEAD_RE.match(x['text'])] for n, i in enumerate(heads): j = heads[n + 1] if n + 1 < len(heads) else len(keep) body = keep[i:j] out.append({'text': ' '.join(' '.join(x['text'] for x in body).split()), 'page': body[0]['page'], 'x': body[0]['x'], 'y': body[0]['y'], 'ids': [id(x) for x in body]}) out.sort(key=lambda n: (n['page'], n['y'], n['x'])) return out # -------------------------------------------------------------------------- # region 5 -- dimension chains on the drawing # -------------------------------------------------------------------------- _CHAIN_TOL = 2.5 def chains(spans, exclude): """Dimension strings grouped into the chains they are printed in. A dimension chain is what a detailer actually edits: a run of segments along one line, adding up to an overall. Grouping by the printed line and comparing the run in order is what turns "two numbers differ somewhere on a D-size sheet" into "the second and third segments of the top chain on the bed plan swapped 3'-3"/1'-0" for 2'-9"/1'-6", total unchanged". `exclude` is the set of span ids already claimed by another region, so a handling-diagram dimension is not also reported as a loose chain. """ dims = [s for s in spans if id(s) not in exclude and parse_dim(s['text']) is not None] groups = [] for vert in (False, True): pool = [s for s in dims if s['vertical'] == vert] used = set() for s in sorted(pool, key=lambda s: (s['page'], s['x'], s['y'])): if id(s) in used: continue axis = 'x' if vert else 'y' run = [o for o in pool if o['page'] == s['page'] and id(o) not in used and abs(o[axis] - s[axis]) <= _CHAIN_TOL] for o in run: used.add(id(o)) run.sort(key=lambda o: o['y'] if vert else o['x']) other = 'y' if vert else 'x' groups.append({ 'page': s['page'], 'vertical': vert, # `pos` is across the run (which line the chain sits on); # `along` is where along that line it starts. Both are needed: # two section details on the same page share an orientation and # can share a `pos`, and matching on `pos` alone paired a # dimension in SECTION A with one in SECTION B and called it a # change on a ticket whose dimension text was byte-identical. 'pos': round(sum(o[axis] for o in run) / float(len(run)), 1), 'along': round(min(o[other] for o in run), 1), 'items': run, }) groups.sort(key=lambda g: (g['page'], g['vertical'], g['pos'])) return groups # -------------------------------------------------------------------------- # region 6 -- callouts and view titles # -------------------------------------------------------------------------- _TITLE_WORDS = ('PLAN', 'SECTION', 'ELEVATION', 'DETAIL', 'VIEW', 'SCHEDULE') def view_titles(spans): """Large-font strings that name a view, used as human-readable locators. The size test alone is not enough: a section bubble's "A", the "MK" tag on the bed plan and the scale string "1/4" = 1'-0"" are all set at the title size, and naming a finding's location "MK, page 1" tells a reader nothing. A view title on these sheets always contains one of the drawing words. """ out = [] for s in spans: if s['size'] < 12.0: continue t = s['text'].upper() if any(w in t for w in _TITLE_WORDS): out.append(s) return out def locate(spans_titles, page, x, y, page_rect=(1224.0, 792.0)): """Name a point on the sheet the way somebody hunting for it would. View title plus a coarse zone. "BED PLAN, page 1" on a D-size sheet is still a lot of paper to search; "BED PLAN, page 1, lower left" is not. """ w, h = page_rect col = 'left' if x < w / 3.0 else ('centre' if x < 2 * w / 3.0 else 'right') row = 'upper' if y < h / 3.0 else ('middle' if y < 2 * h / 3.0 else 'lower') zone = '%s %s' % (row, col) cands = [t for t in spans_titles if t['page'] == page] if not cands: return 'page %d, %s' % (page + 1, zone) best = min(cands, key=lambda t: (t['x'] - x) ** 2 + (t['y'] - y) ** 2) return '%s, page %d, %s' % (best['text'], page + 1, zone) def callouts(spans, claimed): """Everything left over that is a real annotation rather than a dimension. Deliberately excludes anything already claimed by a region and anything that parses as a dimension. What is left is part callouts (P76 W/ SR5602 (S.I.F.)), bar tags, spacing strings, section bubbles. """ out = [] for s in spans: if id(s) in claimed: continue if parse_dim(s['text']) is not None: continue if len(s['text']) < 2: continue out.append(s) return out # -------------------------------------------------------------------------- # top level # -------------------------------------------------------------------------- _TS_RE = re.compile(r'^\d{1,2}/\d{1,2}/\d{2,4}(\s+\d{1,2}:\d{2}(:\d{2})?\s*(AM|PM)?)?$', re.I) def extract(path): """One ticket PDF -> the structure ticketdelta.compare() consumes.""" spans, npages = read_spans(path) haz = hazards(spans, npages) claimed = set() tb = title_block(spans, claimed) bm = bom(spans, claimed) bom_box = None for c in claimed: if isinstance(c, tuple) and c and c[0] == 'bom_bbox': bom_box = {'page': c[1], 'x0': c[2], 'y0': c[3], 'x1': c[4], 'y1': c[5]} claimed = set(c for c in claimed if not isinstance(c, tuple)) hl = handling(spans) nt = notes(spans, claimed) titles = view_titles(spans) for _, dims in hl.items(): for d in dims: claimed.add(id(d)) for s in spans: if s['text'] in HANDLING_LABELS: claimed.add(id(s)) # Note blocks are compared as paragraphs, so their lines must not also # surface as loose callouts. for n in nt: for i in n['ids']: claimed.add(i) ch = chains(spans, claimed) for g in ch: for it in g['items']: claimed.add(id(it)) stamps = [s['text'] for s in spans if _TS_RE.match(s['text'])] return { 'path': path, 'file': os.path.basename(path), 'pages': npages, 'hazards': haz, 'title_block': tb, 'bom': bm, 'bom_bbox': bom_box, 'handling': hl, 'notes': nt, 'chains': ch, 'view_titles': titles, 'callouts': callouts(spans, claimed), 'timestamps': sorted(set(stamps)), 'span_count': len(spans), }