WillTools 2026-09-09 -- clean snapshot of the development repo for review
ci / test (push) Successful in 6m18s
ci / test (push) Successful in 6m18s
History-free snapshot of the private development repository at cbe45c5 (2026-09-09). Code, docs and public catalogue only; no plant or job knowledge layers, no client or job names (tools/scrub.py clean). The intro deck is left out of this snapshot because its slide images could not be checked for job drawings.
This commit is contained in:
@@ -0,0 +1,918 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Full-text index over the knowledge library, one row per PDF page.
|
||||
|
||||
python tools/libindex.py build
|
||||
python tools/libindex.py search "lifting anchor" --limit 5
|
||||
python tools/libindex.py status
|
||||
|
||||
The library is hundreds of pages of source PDFs plus the extraction notes
|
||||
written beside them. Answering "which document, which page, says X" by
|
||||
re-reading PDFs costs minutes every time and is the single most repeated act
|
||||
in `knowledge/library/RUNBOOK.md`. This makes it one query.
|
||||
|
||||
SQLite + FTS5, lexical only. There is no embedding model here and no
|
||||
semantic matching -- a search finds the WORDS you typed, so a miss means
|
||||
"those words are not in the index", never "the library does not say that".
|
||||
The schema leaves room for a vector layer later (`pages.method` already
|
||||
distinguishes how a page's text was obtained) but nothing here pretends to be
|
||||
one.
|
||||
|
||||
WHAT IS IN THE DATABASE, AND WHERE IT LIVES
|
||||
--------------------------------------------
|
||||
`knowledge/library/index.sqlite`, inside a folder that is gitignored as a
|
||||
folder. That is deliberate twice over: the index holds the verbatim text of
|
||||
documents that name producers and jobs, so it may never be committed -- and it
|
||||
is a REBUILDABLE ARTIFACT, so it never needs to be. Delete it and run `build`.
|
||||
|
||||
Identity is the CONTENT HASH, the same convention the ledger keys on: a file
|
||||
that moved is the same document and is not re-read; a file whose bytes changed
|
||||
is a new document and replaces the old rows. Two paths holding identical bytes
|
||||
are one document, and the second is reported rather than duplicated.
|
||||
|
||||
This tool PRINTS the contents of gitignored source material. Its output can
|
||||
carry a producer's name even though this file never does. Do not paste a line
|
||||
of its output into a tracked file; `CLAUDE.md` section 3 has the placeholders.
|
||||
|
||||
Host CPython 3.11+ (modern syntax is fine here, as in `tools/rvt.py`); it is
|
||||
never imported from Revit.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as _dt
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DEFAULT_DB = os.path.join(ROOT, 'knowledge', 'library', 'index.sqlite')
|
||||
DEFAULT_ROOTS = (
|
||||
os.path.join(ROOT, 'knowledge', 'library'),
|
||||
os.path.join(ROOT, 'knowledge', 'MC'),
|
||||
)
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
#: How a page's text was obtained. Only one value exists today; a per-page
|
||||
#: vision transcription would land as a second one ('vision'), which is why
|
||||
#: this is a column and not an assumption. A page with no text layer is
|
||||
#: still 'text-layer' -- we looked, with that method, and there was nothing.
|
||||
METHOD_TEXT_LAYER = 'text-layer'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokenizer
|
||||
# ---------------------------------------------------------------------------
|
||||
# unicode61, diacritics folded, plus exactly three extra token characters:
|
||||
# '#', "'" and '"'. Each one is a character that carries MEANING in this
|
||||
# corpus and that the default tokenizer would throw away:
|
||||
#
|
||||
# #4 a bar size. Without '#', "#4" and a bare "4" are the same
|
||||
# token and every page with a 4 on it answers a bar-size query.
|
||||
# f'c concrete strength. Without "'", it is "f" AND "c".
|
||||
# 8" an inch dimension. Without '"', it is a bare 8.
|
||||
#
|
||||
# Deliberately NOT token characters:
|
||||
#
|
||||
# '.' "7.5" would become one token -- and so would "7.5." at the end
|
||||
# of a sentence, which then no longer matches a search for 7.5.
|
||||
# Decimals are handled on the QUERY side instead: `_match_expr`
|
||||
# turns 7.5 into the phrase "7" "5", which requires the two to be
|
||||
# adjacent, so "value 7 and 5" does not match. Verified both ways
|
||||
# in the tests.
|
||||
# '-' hyphens join words far more often than they join marks, and a
|
||||
# token character would stop "wall" finding "wall-panel".
|
||||
# '_' same reasoning: leaving it a separator is what lets a search
|
||||
# for "anchor" find ANCHOR_FACE_4T.
|
||||
#
|
||||
# Plate marks (P316) and part numbers need nothing special -- unicode61 does
|
||||
# not split a letter/digit boundary.
|
||||
#
|
||||
# The quoting below is load-bearing and easy to get wrong: the value is an SQL
|
||||
# double-quoted string (so the '"' inside it is doubled), and inside THAT the
|
||||
# tokenchars list is an FTS5 single-quoted string (so the "'" inside it is
|
||||
# doubled). FTS5 rejects '[...]' quoting here, whatever the SQL docs suggest.
|
||||
TOKENIZE = '"unicode61 remove_diacritics 2 tokenchars \'#\'\'""\'"'
|
||||
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
|
||||
-- One row per DOCUMENT, keyed by content hash exactly as the ledger is.
|
||||
-- `relpath` is where it was last seen; it is not the identity.
|
||||
CREATE TABLE IF NOT EXISTS docs (
|
||||
hash TEXT PRIMARY KEY,
|
||||
relpath TEXT NOT NULL,
|
||||
pages INTEGER NOT NULL,
|
||||
bytes INTEGER NOT NULL,
|
||||
mtime REAL NOT NULL,
|
||||
ingested_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS docs_relpath ON docs(relpath);
|
||||
|
||||
-- One row per PAGE. An empty `text` with method 'text-layer' means "this
|
||||
-- page was opened and its text layer was empty" -- a scanned image, whose
|
||||
-- words live in the notes until a vision method exists to put them here.
|
||||
CREATE TABLE IF NOT EXISTS pages (
|
||||
doc_hash TEXT NOT NULL,
|
||||
page_no INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
method TEXT NOT NULL,
|
||||
PRIMARY KEY (doc_hash, page_no)
|
||||
);
|
||||
|
||||
-- One row per extraction note. `doc_ref` is the M#/D#/H# id the filename
|
||||
-- starts with, when it has one -- the join back to the ledger's own numbering.
|
||||
CREATE TABLE IF NOT EXISTS notes (
|
||||
relpath TEXT PRIMARY KEY,
|
||||
doc_ref TEXT,
|
||||
text TEXT NOT NULL,
|
||||
mtime REAL NOT NULL,
|
||||
bytes INTEGER NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
ingested_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
|
||||
text,
|
||||
content='pages',
|
||||
content_rowid='rowid',
|
||||
tokenize = {tok}
|
||||
);
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
|
||||
text,
|
||||
content='notes',
|
||||
content_rowid='rowid',
|
||||
tokenize = {tok}
|
||||
);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
|
||||
INSERT INTO pages_fts(rowid, text) VALUES (new.rowid, new.text);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
|
||||
INSERT INTO pages_fts(pages_fts, rowid, text)
|
||||
VALUES ('delete', old.rowid, old.text);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN
|
||||
INSERT INTO pages_fts(pages_fts, rowid, text)
|
||||
VALUES ('delete', old.rowid, old.text);
|
||||
INSERT INTO pages_fts(rowid, text) VALUES (new.rowid, new.text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS notes_ai AFTER INSERT ON notes BEGIN
|
||||
INSERT INTO notes_fts(rowid, text) VALUES (new.rowid, new.text);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS notes_ad AFTER DELETE ON notes BEGIN
|
||||
INSERT INTO notes_fts(notes_fts, rowid, text)
|
||||
VALUES ('delete', old.rowid, old.text);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS notes_au AFTER UPDATE ON notes BEGIN
|
||||
INSERT INTO notes_fts(notes_fts, rowid, text)
|
||||
VALUES ('delete', old.rowid, old.text);
|
||||
INSERT INTO notes_fts(rowid, text) VALUES (new.rowid, new.text);
|
||||
END;
|
||||
""".replace('{tok}', TOKENIZE)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Small helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def use_utf8_stdout() -> None:
|
||||
"""Windows consoles are cp1252 and this corpus is not.
|
||||
|
||||
A page of CJK, or a single typographic dash out of a PDF, raises
|
||||
UnicodeEncodeError on print and the whole run looks like a crash rather
|
||||
than like a result. `errors='replace'` is chosen over `'strict'`
|
||||
deliberately: a character we cannot render is not a reason to lose the
|
||||
hit that contains it.
|
||||
"""
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
stream.reconfigure(encoding='utf-8', errors='replace')
|
||||
except (AttributeError, ValueError): # pragma: no cover - odd host
|
||||
pass
|
||||
|
||||
|
||||
def fts5_available() -> bool:
|
||||
"""True if this interpreter's sqlite3 can build an FTS5 table."""
|
||||
try:
|
||||
con = sqlite3.connect(':memory:')
|
||||
except sqlite3.Error: # pragma: no cover
|
||||
return False
|
||||
try:
|
||||
con.execute('CREATE VIRTUAL TABLE probe USING fts5(x)')
|
||||
return True
|
||||
except sqlite3.Error:
|
||||
return False
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
NO_FTS5 = ('this python has no sqlite3 FTS5 -- the index cannot be built or '
|
||||
'searched on this machine')
|
||||
|
||||
|
||||
def sha256_file(path: str) -> str:
|
||||
"""Streamed: the biggest document in the library is 122 MB."""
|
||||
h = hashlib.sha256()
|
||||
with open(path, 'rb') as f:
|
||||
for block in iter(lambda: f.read(1 << 20), b''):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def relpath(path: str) -> str:
|
||||
"""Repo-relative with forward slashes when it can be; absolute otherwise.
|
||||
|
||||
An extra `--root` may sit anywhere on disk, and forcing a `..\\..\\`
|
||||
relative path onto it would make the stored path unreadable.
|
||||
"""
|
||||
path = os.path.abspath(path)
|
||||
try:
|
||||
rel = os.path.relpath(path, ROOT)
|
||||
except ValueError: # different drive on Windows
|
||||
return path.replace('\\', '/')
|
||||
if rel.startswith('..'):
|
||||
return path.replace('\\', '/')
|
||||
return rel.replace('\\', '/')
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(
|
||||
microsecond=0).isoformat()
|
||||
|
||||
|
||||
#: `M11a-standards-details-p001-085.md` -> `M11a`. A note whose name does not
|
||||
#: start with a ledger id simply has no `doc_ref`; that is not an error, and
|
||||
#: the column is NULL rather than '' so the difference stays visible.
|
||||
_DOC_REF = re.compile(r'^([A-Za-z]{1,2}\d+[A-Za-z]?)[-_]')
|
||||
|
||||
|
||||
def note_doc_ref(name: str) -> str | None:
|
||||
m = _DOC_REF.match(os.path.basename(name))
|
||||
if not m:
|
||||
return None
|
||||
ref = m.group(1)
|
||||
return ref[0].upper() + ref[1:]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Connection / schema
|
||||
# ---------------------------------------------------------------------------
|
||||
def connect(db_path: str, create: bool = False) -> sqlite3.Connection:
|
||||
if not create and not os.path.exists(db_path):
|
||||
raise FileNotFoundError(db_path)
|
||||
parent = os.path.dirname(os.path.abspath(db_path))
|
||||
if create and parent and not os.path.isdir(parent):
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
con = sqlite3.connect(db_path)
|
||||
con.row_factory = sqlite3.Row
|
||||
con.execute('PRAGMA journal_mode = WAL')
|
||||
con.execute('PRAGMA synchronous = NORMAL')
|
||||
return con
|
||||
|
||||
|
||||
def _schema_version(con: sqlite3.Connection) -> int | None:
|
||||
try:
|
||||
row = con.execute(
|
||||
"SELECT value FROM meta WHERE key = 'schema_version'").fetchone()
|
||||
except sqlite3.Error:
|
||||
return None
|
||||
return int(row['value']) if row else None
|
||||
|
||||
|
||||
def ensure_schema(con: sqlite3.Connection, out=print) -> bool:
|
||||
"""Create the schema, wiping first if an older one is in the file.
|
||||
|
||||
Returns True if it wiped. The index is a rebuildable artifact, so a
|
||||
schema change costs a rebuild and never a migration -- but it says so out
|
||||
loud, because a silent wipe and a silent no-op look identical from here.
|
||||
"""
|
||||
wiped = False
|
||||
existing = _schema_version(con)
|
||||
if existing is not None and existing != SCHEMA_VERSION:
|
||||
out('index was built by schema v{0}, this is v{1} -- rebuilding from '
|
||||
'scratch'.format(existing, SCHEMA_VERSION))
|
||||
drop_all(con)
|
||||
wiped = True
|
||||
con.executescript(_SCHEMA)
|
||||
con.execute(
|
||||
"INSERT INTO meta(key, value) VALUES ('schema_version', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(str(SCHEMA_VERSION),))
|
||||
con.commit()
|
||||
return wiped
|
||||
|
||||
|
||||
def drop_all(con: sqlite3.Connection) -> None:
|
||||
for name in ('pages_ai', 'pages_ad', 'pages_au',
|
||||
'notes_ai', 'notes_ad', 'notes_au'):
|
||||
con.execute('DROP TRIGGER IF EXISTS {0}'.format(name))
|
||||
for name in ('pages_fts', 'notes_fts', 'pages', 'notes', 'docs', 'meta'):
|
||||
con.execute('DROP TABLE IF EXISTS {0}'.format(name))
|
||||
con.commit()
|
||||
|
||||
|
||||
def set_meta(con: sqlite3.Connection, key: str, value: str) -> None:
|
||||
con.execute(
|
||||
'INSERT INTO meta(key, value) VALUES (?, ?) '
|
||||
'ON CONFLICT(key) DO UPDATE SET value = excluded.value',
|
||||
(key, str(value)))
|
||||
|
||||
|
||||
def get_meta(con: sqlite3.Connection, key: str) -> str | None:
|
||||
row = con.execute('SELECT value FROM meta WHERE key = ?',
|
||||
(key,)).fetchone()
|
||||
return row['value'] if row else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Walking
|
||||
# ---------------------------------------------------------------------------
|
||||
def find_pdfs(root: str) -> list[str]:
|
||||
found = []
|
||||
for base, dirs, files in os.walk(root):
|
||||
dirs[:] = [d for d in dirs if not d.startswith('.')]
|
||||
for name in sorted(files):
|
||||
if name.lower().endswith('.pdf'):
|
||||
found.append(os.path.join(base, name))
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def find_notes(root: str) -> list[str]:
|
||||
"""Every `*.md` under a folder called `notes`.
|
||||
|
||||
Not every `*.md` under the root: `LEDGER.md` and `RUNBOOK.md` are
|
||||
bookkeeping about the library rather than content out of it, and mixing
|
||||
them into the same ranked list would answer "which document says X" with
|
||||
a row from the index of documents.
|
||||
"""
|
||||
found = []
|
||||
for base, dirs, files in os.walk(root):
|
||||
dirs[:] = [d for d in dirs if not d.startswith('.')]
|
||||
parts = {p.lower() for p in
|
||||
os.path.relpath(base, root).replace('\\', '/').split('/')}
|
||||
if 'notes' not in parts:
|
||||
continue
|
||||
for name in sorted(files):
|
||||
if name.lower().endswith('.md'):
|
||||
found.append(os.path.join(base, name))
|
||||
return sorted(found)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ingest
|
||||
# ---------------------------------------------------------------------------
|
||||
def read_pdf_pages(path: str) -> list[str]:
|
||||
"""Text layer, one string per page. Raises on a PDF that cannot be read."""
|
||||
import pymupdf # imported here so that
|
||||
# `search` and `status` work on a machine with no pymupdf installed.
|
||||
doc = pymupdf.open(path)
|
||||
try:
|
||||
if doc.needs_pass:
|
||||
raise ValueError('encrypted -- no password')
|
||||
return [page.get_text().replace('\r\n', '\n') for page in doc]
|
||||
finally:
|
||||
doc.close()
|
||||
|
||||
|
||||
class BuildReport:
|
||||
"""What one build run did. Printed by `cmd_build`, asserted by the tests."""
|
||||
|
||||
def __init__(self):
|
||||
self.docs_new = 0
|
||||
self.docs_known = 0
|
||||
self.docs_moved = 0
|
||||
self.docs_removed = 0
|
||||
self.pages_added = 0
|
||||
self.pages_removed = 0
|
||||
self.duplicates: list[tuple[str, str]] = []
|
||||
self.unreadable: list[tuple[str, str]] = []
|
||||
self.notes_new = 0
|
||||
self.notes_changed = 0
|
||||
self.notes_known = 0
|
||||
self.notes_removed = 0
|
||||
self.seconds = 0.0
|
||||
|
||||
|
||||
def _under_any(rel: str, root_rels: list[str]) -> bool:
|
||||
for r in root_rels:
|
||||
if rel == r or rel.startswith(r + '/'):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def build(con: sqlite3.Connection, roots: list[str], out=print) -> BuildReport:
|
||||
"""Bring the index in step with what is on disk, under `roots` only.
|
||||
|
||||
Confined to `roots` on purpose. A run with `--root` naming one folder
|
||||
must not delete the rows of a folder it was not asked to look at, or
|
||||
every narrowed re-run would silently gut the index.
|
||||
"""
|
||||
import time
|
||||
started = time.time()
|
||||
rep = BuildReport()
|
||||
stamp = now_iso()
|
||||
|
||||
root_rels = [relpath(r) for r in roots]
|
||||
|
||||
# ---- PDFs -------------------------------------------------------------
|
||||
seen_hashes: dict[str, str] = {}
|
||||
for root in roots:
|
||||
if not os.path.isdir(root):
|
||||
out('root not found, skipped: {0}'.format(relpath(root)))
|
||||
continue
|
||||
for path in find_pdfs(root):
|
||||
rel = relpath(path)
|
||||
try:
|
||||
digest = sha256_file(path)
|
||||
except OSError as exc:
|
||||
rep.unreadable.append((rel, str(exc)))
|
||||
continue
|
||||
if digest in seen_hashes:
|
||||
rep.duplicates.append((rel, seen_hashes[digest]))
|
||||
continue
|
||||
seen_hashes[digest] = rel
|
||||
|
||||
row = con.execute('SELECT relpath, mtime FROM docs WHERE hash = ?',
|
||||
(digest,)).fetchone()
|
||||
if row is not None:
|
||||
rep.docs_known += 1
|
||||
if row['relpath'] != rel:
|
||||
con.execute('UPDATE docs SET relpath = ?, mtime = ? '
|
||||
'WHERE hash = ?',
|
||||
(rel, os.path.getmtime(path), digest))
|
||||
rep.docs_moved += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
texts = read_pdf_pages(path)
|
||||
except Exception as exc: # noqa: BLE001 - any reader
|
||||
rep.unreadable.append((rel, '{0}: {1}'.format(
|
||||
type(exc).__name__, exc)))
|
||||
del seen_hashes[digest]
|
||||
continue
|
||||
|
||||
st = os.stat(path)
|
||||
con.execute(
|
||||
'INSERT INTO docs(hash, relpath, pages, bytes, mtime, '
|
||||
'ingested_at) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
(digest, rel, len(texts), st.st_size, st.st_mtime, stamp))
|
||||
con.executemany(
|
||||
'INSERT INTO pages(doc_hash, page_no, text, method) '
|
||||
'VALUES (?, ?, ?, ?)',
|
||||
[(digest, i + 1, t, METHOD_TEXT_LAYER)
|
||||
for i, t in enumerate(texts)])
|
||||
rep.docs_new += 1
|
||||
rep.pages_added += len(texts)
|
||||
con.commit()
|
||||
|
||||
# Documents that were under a scanned root and are no longer there.
|
||||
gone = [r['hash'] for r in con.execute('SELECT hash, relpath FROM docs')
|
||||
if r['hash'] not in seen_hashes
|
||||
and _under_any(r['relpath'], root_rels)]
|
||||
for digest in gone:
|
||||
n = con.execute('SELECT COUNT(*) c FROM pages WHERE doc_hash = ?',
|
||||
(digest,)).fetchone()['c']
|
||||
con.execute('DELETE FROM pages WHERE doc_hash = ?', (digest,))
|
||||
con.execute('DELETE FROM docs WHERE hash = ?', (digest,))
|
||||
rep.docs_removed += 1
|
||||
rep.pages_removed += n
|
||||
|
||||
# ---- notes ------------------------------------------------------------
|
||||
seen_notes: set[str] = set()
|
||||
for root in roots:
|
||||
if not os.path.isdir(root):
|
||||
continue
|
||||
for path in find_notes(root):
|
||||
rel = relpath(path)
|
||||
seen_notes.add(rel)
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
raw = f.read()
|
||||
except OSError as exc:
|
||||
rep.unreadable.append((rel, str(exc)))
|
||||
seen_notes.discard(rel)
|
||||
continue
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
row = con.execute('SELECT hash FROM notes WHERE relpath = ?',
|
||||
(rel,)).fetchone()
|
||||
if row is not None and row['hash'] == digest:
|
||||
rep.notes_known += 1
|
||||
continue
|
||||
text = raw.decode('utf-8', errors='replace')
|
||||
st = os.stat(path)
|
||||
if row is None:
|
||||
con.execute(
|
||||
'INSERT INTO notes(relpath, doc_ref, text, mtime, bytes, '
|
||||
'hash, ingested_at) VALUES (?, ?, ?, ?, ?, ?, ?)',
|
||||
(rel, note_doc_ref(rel), text, st.st_mtime, st.st_size,
|
||||
digest, stamp))
|
||||
rep.notes_new += 1
|
||||
else:
|
||||
con.execute(
|
||||
'UPDATE notes SET doc_ref = ?, text = ?, mtime = ?, '
|
||||
'bytes = ?, hash = ?, ingested_at = ? WHERE relpath = ?',
|
||||
(note_doc_ref(rel), text, st.st_mtime, st.st_size,
|
||||
digest, stamp, rel))
|
||||
rep.notes_changed += 1
|
||||
con.commit()
|
||||
|
||||
gone_notes = [r['relpath'] for r in con.execute('SELECT relpath FROM notes')
|
||||
if r['relpath'] not in seen_notes
|
||||
and _under_any(r['relpath'], root_rels)]
|
||||
for rel in gone_notes:
|
||||
con.execute('DELETE FROM notes WHERE relpath = ?', (rel,))
|
||||
rep.notes_removed += 1
|
||||
|
||||
rep.seconds = time.time() - started
|
||||
set_meta(con, 'last_build', stamp)
|
||||
set_meta(con, 'last_build_seconds', '{0:.2f}'.format(rep.seconds))
|
||||
set_meta(con, 'last_build_roots', ' | '.join(root_rels))
|
||||
con.commit()
|
||||
return rep
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query building
|
||||
# ---------------------------------------------------------------------------
|
||||
class QueryError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _chunks(query: str) -> list[str]:
|
||||
"""Split a user query into phrase chunks.
|
||||
|
||||
A `"` only opens a quoted phrase at the START of a chunk. That is what
|
||||
lets an inch mark stay attached to its number -- `8"` is one bareword,
|
||||
while `"shear wall"` is one phrase.
|
||||
"""
|
||||
out: list[str] = []
|
||||
i, n = 0, len(query)
|
||||
while i < n:
|
||||
if query[i].isspace():
|
||||
i += 1
|
||||
continue
|
||||
if query[i] == '"':
|
||||
j = query.find('"', i + 1)
|
||||
if j < 0:
|
||||
j = n
|
||||
out.append(query[i + 1:j])
|
||||
i = j + 1
|
||||
else:
|
||||
j = i
|
||||
while j < n and not query[j].isspace():
|
||||
j += 1
|
||||
out.append(query[i:j])
|
||||
i = j
|
||||
return [c for c in out if c.strip()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Thousands-separator equivalence (query side)
|
||||
# ---------------------------------------------------------------------------
|
||||
# The defect this exists for: `7,690` is a published capacity that appears in
|
||||
# the source PDFs and the notes ALWAYS with the comma (that is how the
|
||||
# catalogue prints it), so searching the bare digits `7690` -- which is how
|
||||
# anyone asking "where does 7690 come from" actually types it -- returned 0
|
||||
# hits although the corpus holds the fact on four pages and in two notes. A
|
||||
# search that cannot find a number it holds, and reports that as 0 rather
|
||||
# than an error, is indistinguishable from "not in the corpus" and is exactly
|
||||
# the failure this index exists to prevent (CLAUDE.md section on "could not
|
||||
# judge" vs "judged, found nothing").
|
||||
#
|
||||
# Why query-side rather than a stored, normalised copy: the comma is not a
|
||||
# tokenchar (see the TOKENIZE comment above), so `unicode61` already splits
|
||||
# `7,690` into the two adjacent tokens "7" and "690" at INDEX time -- that is
|
||||
# how `"7,690"` (quoted, comma and all) already found those hits before this
|
||||
# fix. The only missing half is the reverse: a bare `7690` tokenizes to the
|
||||
# single token "7690", which never occurs in a corpus that always writes the
|
||||
# comma. So nothing about how text is stored needs to change; the query just
|
||||
# has to offer FTS5 both spellings and let it match either. That costs no
|
||||
# rebuild and touches every existing row immediately, at the price of only
|
||||
# helping searches that go through `match_expr` (not `--raw`, which hands the
|
||||
# string to FTS5 untouched on purpose).
|
||||
#
|
||||
# Deliberately narrow triggers, so this never reaches past the number:
|
||||
# * A chunk with a '.' is left alone -- decimals already work today via the
|
||||
# adjacent-phrase trick above, and `3.5` / `4.0` / `0.6` must keep
|
||||
# behaving exactly as they do now. This function never sees them: the
|
||||
# caller checks for '.' first.
|
||||
# * A chunk containing anything but digits and commas (`QL589G`, a bar mark
|
||||
# `SR5602`, a control number `25008-40085`) is left untouched -- the
|
||||
# letters or the hyphen already fail `_BARE_DIGITS` / `_GROUPED_DIGITS`
|
||||
# below, so a part number is never mistaken for a grouped integer.
|
||||
# * Grouping is only offered from 4 digits up (`1,000` is the shortest real
|
||||
# grouped number); `4`, `#4`, `123` do not get a comma variant nobody
|
||||
# would ever type.
|
||||
_BARE_DIGITS = re.compile(r'^\d{4,}$')
|
||||
_GROUPED_DIGITS = re.compile(r'^\d{1,3}(,\d{3})+$')
|
||||
|
||||
|
||||
def _group_thousands(digits: str) -> str:
|
||||
"""`'7690'` -> `'7,690'`. Standard right-to-left grouping of 3."""
|
||||
rev = digits[::-1]
|
||||
groups = [rev[i:i + 3] for i in range(0, len(rev), 3)]
|
||||
return ','.join(groups)[::-1]
|
||||
|
||||
|
||||
def _numeric_variants(chunk: str) -> list[str] | None:
|
||||
"""Both spellings of a thousands-grouped integer, or `None`.
|
||||
|
||||
Returns `[bare, grouped]` when `chunk` is unambiguously an integer typed
|
||||
either way (`'7690'` or `'7,690'`); `None` for anything else, which tells
|
||||
the caller to fall back to quoting `chunk` exactly as it always has.
|
||||
"""
|
||||
if _BARE_DIGITS.match(chunk):
|
||||
grouped = _group_thousands(chunk)
|
||||
return [chunk, grouped] if grouped != chunk else None
|
||||
if _GROUPED_DIGITS.match(chunk):
|
||||
return [chunk.replace(',', ''), chunk]
|
||||
return None
|
||||
|
||||
|
||||
def match_expr(query: str, raw: bool = False) -> str:
|
||||
"""User text -> an FTS5 MATCH expression.
|
||||
|
||||
Every chunk becomes a quoted phrase, so nothing the user types is read as
|
||||
FTS5 operator syntax by accident and `7.5` becomes the adjacent pair
|
||||
"7" "5" rather than a parse error. A trailing `*` on a bareword survives
|
||||
as an FTS5 prefix. `--raw` hands the string over untouched, for the rare
|
||||
query that really wants OR / NEAR / column filters.
|
||||
|
||||
A chunk that is a bare or comma-grouped integer of 4+ digits (`7690` or
|
||||
`7,690`) expands into an OR of both spellings, so a search finds a number
|
||||
regardless of which way the source printed it. See `_numeric_variants`.
|
||||
"""
|
||||
if raw:
|
||||
if not query.strip():
|
||||
raise QueryError('empty query')
|
||||
return query
|
||||
parts = []
|
||||
for chunk in _chunks(query):
|
||||
prefix = chunk.endswith('*') and len(chunk) > 1
|
||||
if prefix:
|
||||
chunk = chunk[:-1]
|
||||
if not chunk.strip():
|
||||
continue
|
||||
variants = None if prefix else _numeric_variants(chunk)
|
||||
if variants:
|
||||
parts.append('({0})'.format(' OR '.join(
|
||||
'"{0}"'.format(v.replace('"', '""')) for v in variants)))
|
||||
else:
|
||||
parts.append('"{0}"{1}'.format(chunk.replace('"', '""'),
|
||||
'*' if prefix else ''))
|
||||
if not parts:
|
||||
raise QueryError('empty query')
|
||||
return ' '.join(parts)
|
||||
|
||||
|
||||
_WS = re.compile(r'\s+')
|
||||
|
||||
|
||||
def flatten(text: str, limit: int = 0) -> str:
|
||||
"""PDF text is full of newlines; a terminal hit should be one line."""
|
||||
flat = _WS.sub(' ', text).strip()
|
||||
if limit and len(flat) > limit:
|
||||
flat = flat[:limit - 3] + '...'
|
||||
return flat
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Search
|
||||
# ---------------------------------------------------------------------------
|
||||
def search_pages(con, expr, limit, doc=None):
|
||||
sql = ["SELECT d.relpath, d.hash, p.page_no, bm25(pages_fts) AS score,",
|
||||
" snippet(pages_fts, 0, '[', ']', ' ... ', 14) AS snip",
|
||||
"FROM pages_fts",
|
||||
"JOIN pages p ON p.rowid = pages_fts.rowid",
|
||||
"JOIN docs d ON d.hash = p.doc_hash",
|
||||
"WHERE pages_fts MATCH ?"]
|
||||
args: list = [expr]
|
||||
if doc:
|
||||
sql.append('AND (d.hash LIKE ? OR d.relpath LIKE ?)')
|
||||
args += [doc.lower() + '%', '%' + doc + '%']
|
||||
sql.append('ORDER BY score LIMIT ?')
|
||||
args.append(limit)
|
||||
return con.execute('\n'.join(sql), args).fetchall()
|
||||
|
||||
|
||||
def search_notes(con, expr, limit, doc=None):
|
||||
sql = ["SELECT n.relpath, n.doc_ref, bm25(notes_fts) AS score,",
|
||||
" snippet(notes_fts, 0, '[', ']', ' ... ', 14) AS snip",
|
||||
"FROM notes_fts",
|
||||
"JOIN notes n ON n.rowid = notes_fts.rowid",
|
||||
"WHERE notes_fts MATCH ?"]
|
||||
args: list = [expr]
|
||||
if doc:
|
||||
sql.append('AND (IFNULL(n.doc_ref, "") LIKE ? OR n.relpath LIKE ?)')
|
||||
args += [doc + '%', '%' + doc + '%']
|
||||
sql.append('ORDER BY score LIMIT ?')
|
||||
args.append(limit)
|
||||
return con.execute('\n'.join(sql), args).fetchall()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Commands
|
||||
# ---------------------------------------------------------------------------
|
||||
def cmd_build(args, out=print) -> int:
|
||||
roots = [os.path.abspath(r) for r in (args.root or DEFAULT_ROOTS)]
|
||||
con = connect(args.db, create=True)
|
||||
try:
|
||||
if args.rebuild:
|
||||
drop_all(con)
|
||||
out('--rebuild: dropped every table, starting empty')
|
||||
ensure_schema(con, out=out)
|
||||
rep = build(con, roots, out=out)
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
out('roots: ' + ', '.join(relpath(r) for r in roots))
|
||||
out('docs: {0} new, {1} unchanged, {2} moved, {3} removed'.format(
|
||||
rep.docs_new, rep.docs_known, rep.docs_moved, rep.docs_removed))
|
||||
out('pages: +{0}, -{1}'.format(rep.pages_added, rep.pages_removed))
|
||||
out('notes: {0} new, {1} changed, {2} unchanged, {3} removed'.format(
|
||||
rep.notes_new, rep.notes_changed, rep.notes_known, rep.notes_removed))
|
||||
for rel, first in rep.duplicates:
|
||||
out('duplicate content, not indexed twice: {0} == {1}'.format(
|
||||
rel, first))
|
||||
for rel, why in rep.unreadable:
|
||||
out('COULD NOT READ (not indexed, will be retried next build): '
|
||||
'{0} -- {1}'.format(rel, why))
|
||||
out('built in {0:.2f}s'.format(rep.seconds))
|
||||
return 0
|
||||
|
||||
|
||||
def _open_for_reading(db: str, out=print):
|
||||
if not os.path.exists(db):
|
||||
out('no index built yet -- run: python tools/libindex.py build')
|
||||
return None
|
||||
con = connect(db)
|
||||
if _schema_version(con) != SCHEMA_VERSION:
|
||||
con.close()
|
||||
out('index is from another schema version -- run: '
|
||||
'python tools/libindex.py build')
|
||||
return None
|
||||
return con
|
||||
|
||||
|
||||
def cmd_search(args, out=print) -> int:
|
||||
con = _open_for_reading(args.db, out=out)
|
||||
if con is None:
|
||||
return 1
|
||||
try:
|
||||
try:
|
||||
expr = match_expr(args.query, raw=args.raw)
|
||||
except QueryError as exc:
|
||||
out('bad query -- {0}'.format(exc))
|
||||
return 1
|
||||
try:
|
||||
hits = []
|
||||
if args.kind in ('all', 'pages'):
|
||||
hits += [('page', r)
|
||||
for r in search_pages(con, expr, args.limit, args.doc)]
|
||||
if args.kind in ('all', 'notes'):
|
||||
hits += [('note', r)
|
||||
for r in search_notes(con, expr, args.limit, args.doc)]
|
||||
except sqlite3.OperationalError as exc:
|
||||
# A malformed FTS5 expression is "could not judge", not "nothing
|
||||
# matched", so it must never print 0 hits.
|
||||
out('could not run that query -- {0}'.format(exc))
|
||||
out('(the query was handed to FTS5 as: {0})'.format(expr))
|
||||
return 1
|
||||
|
||||
hits.sort(key=lambda h: h[1]['score'])
|
||||
hits = hits[:args.limit]
|
||||
|
||||
out('query: {0}'.format(expr))
|
||||
if not hits:
|
||||
out('0 hits')
|
||||
return 0
|
||||
out('{0} hit(s), best first (bm25: lower is better):'.format(len(hits)))
|
||||
for kind, row in hits:
|
||||
if kind == 'page':
|
||||
where = '{0} p.{1}'.format(row['relpath'], row['page_no'])
|
||||
tag = row['hash'][:8]
|
||||
else:
|
||||
where = row['relpath']
|
||||
tag = row['doc_ref'] or 'note'
|
||||
out('')
|
||||
out(' [{0}] {1} bm25 {2:.2f} {3}'.format(
|
||||
kind, where, row['score'], tag))
|
||||
out(' {0}'.format(flatten(row['snip'], 400)))
|
||||
return 0
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def cmd_status(args, out=print) -> int:
|
||||
con = _open_for_reading(args.db, out=out)
|
||||
if con is None:
|
||||
return 1
|
||||
def one(sql):
|
||||
return con.execute(sql).fetchone()[0]
|
||||
|
||||
docs = one('SELECT COUNT(*) FROM docs')
|
||||
pages = one('SELECT COUNT(*) FROM pages')
|
||||
empty = one("SELECT COUNT(*) FROM pages WHERE TRIM(text) = ''")
|
||||
notes = one('SELECT COUNT(*) FROM notes')
|
||||
blank_docs = one(
|
||||
"SELECT COUNT(*) FROM (SELECT doc_hash FROM pages GROUP BY doc_hash "
|
||||
"HAVING SUM(CASE WHEN TRIM(text) = '' THEN 0 ELSE 1 END) = 0)")
|
||||
out('index: {0}'.format(relpath(args.db)))
|
||||
out('docs: {0}'.format(docs))
|
||||
out('pages: {0} ({1} with an empty text layer'.format(pages, empty)
|
||||
+ (', {0} document(s) empty throughout -- scanned, '
|
||||
'their words are in the notes only)'.format(blank_docs)
|
||||
if blank_docs else ')'))
|
||||
out('notes: {0}'.format(notes))
|
||||
roots_meta = get_meta(con, 'last_build_roots') or ''
|
||||
root_rels = [r for r in roots_meta.split(' | ') if r]
|
||||
out('roots: {0}'.format(roots_meta or '(none)'))
|
||||
|
||||
# `last_build_roots` is only the ROOTS NAMED ON THE MOST RECENT BUILD --
|
||||
# not a history of everywhere the index has ever been told to look. A
|
||||
# doc or note ingested by an earlier build with a different --root is
|
||||
# never removed by a later build that does not name that root (`build`'s
|
||||
# own docstring: "must not delete the rows of a folder it was not asked
|
||||
# to look at"), so it stays in the database forever while this line goes
|
||||
# on describing only the last build's roots. That is `status` UNDER-
|
||||
# REPORTING what the index actually holds -- the same silence-as-success
|
||||
# shape as a search returning 0 hits for a reason other than "not
|
||||
# found" -- so state it explicitly rather than let the line above imply
|
||||
# the database contains nothing else.
|
||||
all_rels = [r['relpath'] for r in con.execute(
|
||||
'SELECT relpath FROM docs UNION SELECT relpath FROM notes')]
|
||||
outside = sorted(r for r in all_rels if not _under_any(r, root_rels))
|
||||
if outside:
|
||||
out(' ** {0} of those item(s) live OUTSIDE the roots above --'
|
||||
.format(len(outside)))
|
||||
out(' ** from an earlier build with a different --root; still '
|
||||
'resident because no later')
|
||||
out(' ** build has scanned (and could therefore remove) them. '
|
||||
'The database holds more')
|
||||
out(' ** than the "roots" line says. Examples:')
|
||||
for r in outside[:8]:
|
||||
out(' {0}'.format(r))
|
||||
if len(outside) > 8:
|
||||
out(' ... and {0} more'.format(len(outside) - 8))
|
||||
out('built: {0} in {1}s'.format(get_meta(con, 'last_build') or '(never)',
|
||||
get_meta(con, 'last_build_seconds')
|
||||
or '?'))
|
||||
con.close()
|
||||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
def make_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(
|
||||
prog='libindex',
|
||||
description='Page-granular full-text index over the knowledge '
|
||||
'library (SQLite + FTS5, lexical only).')
|
||||
sub = p.add_subparsers(dest='cmd')
|
||||
|
||||
b = sub.add_parser('build', help='ingest / re-ingest; safe to re-run')
|
||||
b.add_argument('--root', action='append',
|
||||
help='folder to scan; repeatable. Default: '
|
||||
'knowledge/library and knowledge/MC')
|
||||
b.add_argument('--db', default=DEFAULT_DB)
|
||||
b.add_argument('--rebuild', action='store_true',
|
||||
help='throw the index away and start from empty')
|
||||
|
||||
s = sub.add_parser('search', help='find pages and notes')
|
||||
s.add_argument('query')
|
||||
s.add_argument('--limit', type=int, default=10)
|
||||
s.add_argument('--doc', help='hash prefix, ledger id, or path substring')
|
||||
s.add_argument('--kind', choices=('all', 'pages', 'notes'), default='all')
|
||||
s.add_argument('--raw', action='store_true',
|
||||
help='pass the query to FTS5 verbatim (OR / NEAR / *)')
|
||||
s.add_argument('--db', default=DEFAULT_DB)
|
||||
|
||||
t = sub.add_parser('status', help='what the index currently holds')
|
||||
t.add_argument('--db', default=DEFAULT_DB)
|
||||
return p
|
||||
|
||||
|
||||
def main(argv=None, out=print) -> int:
|
||||
use_utf8_stdout()
|
||||
parser = make_parser()
|
||||
args = parser.parse_args(argv)
|
||||
if not args.cmd:
|
||||
parser.print_help()
|
||||
return 1
|
||||
if not fts5_available():
|
||||
out(NO_FTS5)
|
||||
return 1
|
||||
return {'build': cmd_build,
|
||||
'search': cmd_search,
|
||||
'status': cmd_status}[args.cmd](args, out=out)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user