willtools-demo/WillTools.extension/tests/test_libindex.py

# -*- coding: utf-8 -*-
"""Tests for tools/libindex.py -- the library full-text index.

`libindex.py` is host CPython, so it is loaded the way `test_scrub.py` and
`test_knowledge_drift.py` load their tools: put `tools/` on `sys.path` and
import it. It lives here rather than beside `tools/` because this is where
every other test of a `tools/` module already lives, and one `python -m
pytest` from the repo root has to reach all of them.

WHAT IS WORTH TESTING HERE
---------------------------
Not "does FTS5 work". Two things:

  * **The tokenizer decision.** The whole value of the index rests on `#4`
    not being the same token as `4`, on `f'c` surviving as one word, on an
    inch mark staying attached to its number, and on a decimal like `7.5`
    matching without also matching a page that merely has a 7 and a 5 on it.
    Each of those is a separate assertion below, because each is a separate
    way the tokenizer choice could be wrong.

  * **Incremental re-ingest.** Other agents write notes into the library
    while a build is running, so `build` is re-run constantly and every
    re-run has to be cheap AND correct: unchanged content skipped, changed
    content replaced (not doubled), a moved file recognised as the same
    document, a vanished file's rows removed -- and a build given one `--root`
    must not delete the rows belonging to a root it was not asked about.

Plus the error discipline: "0 hits" and "no index built yet" are different
sentences and one must never be printed for the other.

Every PDF here is built with pymupdf into `tmp_path` and every note is
written there too. Nothing reads the real library, and nothing writes into
the working directory. No client, job or producer name appears; the
documents are invented and the marks are placeholders.
"""
# pylint: disable=import-error,invalid-name,redefined-outer-name
import os
import sqlite3
import sys

import pytest

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
    os.path.abspath(__file__))))
TOOLS = os.path.join(ROOT, 'tools')
if TOOLS not in sys.path:
    sys.path.insert(0, TOOLS)

libindex = pytest.importorskip('libindex')
pymupdf = pytest.importorskip('pymupdf')


# ---------------------------------------------------------------------------
# fixtures / helpers
# ---------------------------------------------------------------------------
def make_pdf(path, pages):
    """A tiny real PDF: one page per string, '' meaning no text layer."""
    doc = pymupdf.open()
    for text in pages:
        page = doc.new_page()
        if text:
            page.insert_text((72, 72), text, fontsize=11)
    doc.save(str(path))
    doc.close()
    return str(path)


class Run:
    """One CLI invocation: its exit code and every line it printed."""

    def __init__(self, code, lines):
        self.code = code
        self.lines = lines
        self.text = '\n'.join(lines)

    def __repr__(self):                            # pragma: no cover - debug
        return 'Run({0})\n{1}'.format(self.code, self.text)


def run(*argv):
    lines = []
    code = libindex.main(list(argv), out=lambda *a: lines.append(
        ' '.join(str(x) for x in a)))
    return Run(code, lines)


@pytest.fixture()
def lib(tmp_path):
    """An empty stand-in library: a drop folder with a notes/ subfolder."""
    root = tmp_path / 'lib'
    (root / 'notes').mkdir(parents=True)
    return root


@pytest.fixture()
def db(tmp_path):
    return str(tmp_path / 'index.sqlite')


def rows(db_path, sql, args=()):
    con = sqlite3.connect(db_path)
    try:
        return con.execute(sql, args).fetchall()
    finally:
        con.close()


def one(db_path, sql, args=()):
    return rows(db_path, sql, args)[0][0]


# ---------------------------------------------------------------------------
# availability
# ---------------------------------------------------------------------------
def test_fts5_is_available_on_this_interpreter():
    # The tool refuses rather than falling back to LIKE, so if this ever goes
    # false the refusal is the correct behaviour -- but on CPython 3.12 for
    # Windows it is true, and the rest of this file assumes it.
    assert libindex.fts5_available() is True


def test_the_refusal_sentence_names_fts5_and_does_not_offer_a_fallback():
    assert 'FTS5' in libindex.NO_FTS5
    assert 'LIKE' not in libindex.NO_FTS5


# ---------------------------------------------------------------------------
# building
# ---------------------------------------------------------------------------
def test_build_indexes_every_page_and_every_note(lib, db):
    make_pdf(lib / 'guide.pdf', ['page one text', 'page two text'])
    (lib / 'notes' / 'M4-guide.md').write_text(
        '# guide\n\nextracted facts live here\n', encoding='utf-8')

    r = run('build', '--root', str(lib), '--db', db)
    assert r.code == 0

    assert one(db, 'SELECT COUNT(*) FROM docs') == 1
    assert one(db, 'SELECT COUNT(*) FROM pages') == 2
    assert one(db, 'SELECT COUNT(*) FROM notes') == 1
    assert one(db, 'SELECT pages FROM docs') == 2
    # The identity is the content hash, matching the ledger's convention.
    assert len(one(db, 'SELECT hash FROM docs')) == 64


def test_a_page_with_no_text_layer_is_stored_empty_and_still_text_layer(
        lib, db):
    # Not skipped, and not marked with some other method: we DID look at it,
    # with the text-layer method, and it had nothing. A later per-page vision
    # pass is what would give it a different method value.
    make_pdf(lib / 'scan.pdf', ['readable page', '', ''])
    run('build', '--root', str(lib), '--db', db)

    assert one(db, 'SELECT COUNT(*) FROM pages') == 3
    methods = {m for (m,) in rows(db, 'SELECT DISTINCT method FROM pages')}
    assert methods == {libindex.METHOD_TEXT_LAYER}
    assert one(db, "SELECT COUNT(*) FROM pages WHERE TRIM(text) = ''") == 2


def test_status_reports_the_empty_text_pages_rather_than_hiding_them(lib, db):
    make_pdf(lib / 'scan.pdf', ['readable', '', ''])
    run('build', '--root', str(lib), '--db', db)
    r = run('status', '--db', db)
    assert r.code == 0
    assert 'docs:   1' in r.text
    assert 'pages:  3' in r.text
    assert '2 with an empty text layer' in r.text
    assert 'notes:  0' in r.text
    assert 'built:' in r.text


def test_a_pdf_that_cannot_be_read_is_named_and_not_silently_dropped(lib, db):
    (lib / 'broken.pdf').write_bytes(b'this is not a pdf at all')
    make_pdf(lib / 'fine.pdf', ['good page'])

    r = run('build', '--root', str(lib), '--db', db)
    assert r.code == 0
    assert 'COULD NOT READ' in r.text
    assert 'broken.pdf' in r.text
    # and the readable one still got in
    assert one(db, 'SELECT COUNT(*) FROM docs') == 1

    # not recorded as a document, so a later fix is picked up rather than
    # being treated as already-known
    r2 = run('build', '--root', str(lib), '--db', db)
    assert 'COULD NOT READ' in r2.text


def test_notes_are_taken_from_the_notes_folder_and_not_from_the_ledger(
        lib, db):
    (lib / 'LEDGER.md').write_text('| # | sha256 | file |\n', encoding='utf-8')
    (lib / 'RUNBOOK.md').write_text('how to drain the folder\n',
                                    encoding='utf-8')
    (lib / 'notes' / 'M4-guide.md').write_text('a real note\n',
                                               encoding='utf-8')
    run('build', '--root', str(lib), '--db', db)
    got = {os.path.basename(p) for (p,) in rows(db, 'SELECT relpath FROM notes')}
    assert got == {'M4-guide.md'}


# ---------------------------------------------------------------------------
# doc_ref parsing
# ---------------------------------------------------------------------------
@pytest.mark.parametrize('name,expected', [
    ('M1-brd-memo.md', 'M1'),
    ('M11a-details-p001-085.md', 'M11a'),
    ('D2-datasheet.md', 'D2'),
    ('H1-drawing-set.md', 'H1'),
    ('m3-lowercase.md', 'M3'),
    ('notes-about-things.md', None),
    ('README.md', None),
])
def test_note_doc_ref_is_the_ledger_id_or_nothing(name, expected):
    # NULL, not '' -- "this note names no ledger row" and "this note names
    # the empty row" are not the same claim.
    assert libindex.note_doc_ref(name) == expected


def test_doc_ref_lands_in_the_row_and_is_null_when_absent(lib, db):
    (lib / 'notes' / 'M4-guide.md').write_text('one\n', encoding='utf-8')
    (lib / 'notes' / 'scratch-thoughts.md').write_text('two\n',
                                                       encoding='utf-8')
    run('build', '--root', str(lib), '--db', db)
    got = dict((os.path.basename(p), ref) for p, ref in
               rows(db, 'SELECT relpath, doc_ref FROM notes'))
    assert got['M4-guide.md'] == 'M4'
    assert got['scratch-thoughts.md'] is None


# ---------------------------------------------------------------------------
# the tokenizer decision
# ---------------------------------------------------------------------------
@pytest.fixture()
def shapes(lib, db):
    """One library holding the four token shapes that motivated the choice."""
    make_pdf(lib / 'shapes.pdf', [
        "reinforce with #4 bars where f'c is 5000 psi",
        'plate mark P316 sits at 7.5 inches from the edge',
        'a bare 4 appears here, and a 7 and a 5 far apart',
        'slab is 8" thick at the bearing',
    ])
    run('build', '--root', str(lib), '--db', db)
    return db


def hits(db_path, query, *extra):
    r = run('search', query, '--db', db_path, *extra)
    assert r.code == 0, r.text
    pages = [int(line.split('p.')[1].split()[0])
             for line in r.lines if line.strip().startswith('[page]')]
    return r, pages


def test_a_bar_size_is_not_the_same_token_as_a_bare_number(shapes):
    # This is the whole reason '#' is a token character. Without it, every
    # page with a 4 on it answers a query for #4.
    _, p = hits(shapes, '#4')
    assert p == [1]
    _, p = hits(shapes, '4')
    assert p == [3]


def test_concrete_strength_survives_its_apostrophe(shapes):
    _, p = hits(shapes, "f'c")
    assert p == [1]


def test_an_inch_mark_stays_attached_to_its_number(shapes):
    _, p = hits(shapes, '8"')
    assert p == [4]


def test_a_plate_mark_needs_no_special_handling(shapes):
    _, p = hits(shapes, 'P316')
    assert p == [2]


def test_a_decimal_matches_its_page_and_not_a_scattered_seven_and_five(shapes):
    # '.' is deliberately NOT a token character (a trailing sentence period
    # would glue itself on). The decimal is handled as an adjacent phrase
    # instead, which is why page 3 -- which has a 7 and a 5, far apart --
    # must not match.
    r, p = hits(shapes, '7.5')
    assert p == [2]
    # It goes to FTS5 as a quoted PHRASE -- the quoting is what turns the
    # decimal into an adjacency requirement instead of a parse error.
    assert 'query: "7.5"' in r.text


# ---------------------------------------------------------------------------
# thousands-separator equivalence (the 7690 / 7,690 defect)
# ---------------------------------------------------------------------------
# Pinned BEFORE this fix existed, so a future change to the decimal path
# cannot silently regress it: '.' is not a tokenchar, a decimal is still
# handled purely by adjacent-phrase quoting, and none of that machinery may
# be disturbed by the comma-equivalence feature below.
@pytest.mark.parametrize('query,expr', [
    ('3.5', '"3.5"'),
    ('4.0', '"4.0"'),
    ('0.6', '"0.6"'),
    ('7.5', '"7.5"'),
])
def test_decimal_match_expr_is_pinned_and_unchanged_by_the_comma_fix(
        query, expr):
    assert libindex.match_expr(query) == expr


@pytest.fixture()
def commas(lib, db):
    """A library that prints the SAME fact only with a thousands separator.

    This is the actual shape of the defect: `ALP Catalog 2024` prints
    `7,690` -- never the bare digits -- on two pages and in a note, and the
    2026-09-01 incident was a query for the bare `7690` coming back 0 hits
    although the fact was sitting right there. `16,000` sits beside it on
    the same page (a different capacity, from CLAUDE.md section 3's own
    contradiction story) so the fix can be proven in both digit-count
    directions without inventing numbers foreign to this corpus.
    """
    make_pdf(lib / 'ALPCatalog2024.pdf', [
        'irrelevant cover page',
        'erection anchor capacity 8" 6,030 4,010 7,690 tension rating',
        'a different table entirely: 16,000 lb working load, QL185G',
    ])
    (lib / 'notes' / 'ALP-2024-erection-anchors.md').write_text(
        '| size | tension |\n| 8T | 7,690 |\n| 8T-12T | 16,000 |\n',
        encoding='utf-8')
    run('build', '--root', str(lib), '--db', db)
    return db


def test_bare_digits_find_a_number_the_corpus_only_prints_with_a_comma(
        commas):
    # This is the exact defect: before the fix this returned 0 hits even
    # though the fact is on page 2 and in the note.
    r, p = hits(commas, '7690')
    assert r.code == 0
    assert p == [2]
    assert '[note]' in r.text
    assert '0 hits' not in r.text


def test_comma_digits_still_find_a_number_the_corpus_prints_bare(lib, db):
    # The reverse direction: a source that prints the number WITHOUT a
    # separator must still answer a comma-typed query.
    make_pdf(lib / 'bare.pdf', ['working load 16000 lbs rated'])
    run('build', '--root', str(lib), '--db', db)
    _, p = hits(db, '16,000')
    assert p == [1]


def test_16000_and_16_comma_000_return_the_same_hits(commas):
    a = hits(commas, '16000')[1]
    b = hits(commas, '16,000')[1]
    assert a == b == [3]


def test_7690_and_7_comma_690_return_the_same_hits(commas):
    a = hits(commas, '7690')[1]
    b = hits(commas, '7,690')[1]
    assert a == b == [2]


def test_a_part_number_is_not_treated_as_a_grouped_integer(lib, db):
    make_pdf(lib / 'parts.pdf', [
        "the QL589G anchor and control number 25008-40085, bar mark SR5602"])
    run('build', '--root', str(lib), '--db', db)
    for query in ('QL589G', '25008-40085', 'SR5602'):
        _, p = hits(db, query)
        assert p == [1], query


@pytest.mark.parametrize('query,expr', [
    ('7690', '("7690" OR "7,690")'),
    ('7,690', '("7690" OR "7,690")'),
    ('16000', '("16000" OR "16,000")'),
    ('16,000', '("16000" OR "16,000")'),
])
def test_match_expr_expands_a_grouped_integer_both_ways(query, expr):
    assert libindex.match_expr(query) == expr


@pytest.mark.parametrize('query', [
    'QL589G', '25008-40085', 'SR5602', '#4', '123', "f'c", '8"',
])
def test_match_expr_does_not_expand_non_numeric_or_short_chunks(query):
    expr = libindex.match_expr(query)
    assert ' OR ' not in expr


def test_a_multi_word_query_is_and_not_a_phrase(shapes):
    _, p = hits(shapes, 'plate edge')
    assert p == [2]


def test_a_quoted_query_is_one_phrase(shapes):
    _, p = hits(shapes, '"plate mark P316"')
    assert p == [2]
    _, p = hits(shapes, '"P316 plate mark"')      # wrong order, no phrase
    assert p == []


def test_underscores_and_hyphens_stay_separators_so_recall_survives(lib, db):
    make_pdf(lib / 'ids.pdf', ['the ANCHOR_FACE_4T detail on a wall-panel'])
    run('build', '--root', str(lib), '--db', db)
    for query in ('anchor', 'wall', 'panel', 'ANCHOR_FACE_4T'):
        _, p = hits(db, query)
        assert p == [1], query


def test_a_prefix_star_still_reaches_fts5(shapes):
    _, p = hits(shapes, 'reinforc*')
    assert p == [1]


def test_cjk_is_indexed_and_printable_on_a_cp1252_console(lib, db):
    # The encoding trap from CLAUDE.md section 7: a Chinese hit must come back
    # as a hit, not as a UnicodeEncodeError that looks like a crash.
    (lib / 'notes' / 'M9-mixed.md').write_text(
        u'plate P316 -- 应力极限 is the governing limit\n', encoding='utf-8')
    run('build', '--root', str(lib), '--db', db)
    r = run('search', u'应力极限', '--db', db)
    assert r.code == 0
    assert '1 hit(s)' in r.text
    assert 'M9-mixed.md' in r.text


# ---------------------------------------------------------------------------
# match_expr, at the unit level
# ---------------------------------------------------------------------------
@pytest.mark.parametrize('query,expr', [
    ('7.5', '"7.5"'),
    ("f'c", '"f\'c"'),
    ('#4', '"#4"'),
    ('8"', '"8"""'),
    ('shear wall', '"shear" "wall"'),
    ('"shear wall"', '"shear wall"'),
    ('beam*', '"beam"*'),
    ('  spaced   out  ', '"spaced" "out"'),
])
def test_match_expr_quotes_every_chunk(query, expr):
    assert libindex.match_expr(query) == expr


def test_match_expr_passes_raw_through_untouched():
    assert libindex.match_expr('a OR b', raw=True) == 'a OR b'


def test_match_expr_refuses_an_empty_query():
    with pytest.raises(libindex.QueryError):
        libindex.match_expr('   ')


def test_raw_lets_an_or_query_through(shapes):
    _, p = hits(shapes, 'P316 OR bearing', '--raw')
    assert sorted(p) == [2, 4]


# ---------------------------------------------------------------------------
# error discipline: could not judge vs judged and found nothing
# ---------------------------------------------------------------------------
def test_zero_hits_and_no_index_are_different_sentences(lib, db):
    make_pdf(lib / 'guide.pdf', ['something entirely else'])
    run('build', '--root', str(lib), '--db', db)

    found_nothing = run('search', 'unobtainium', '--db', db)
    assert found_nothing.code == 0
    assert '0 hits' in found_nothing.text
    assert 'no index built yet' not in found_nothing.text

    missing = run('search', 'unobtainium', '--db', db + '.nope')
    assert missing.code == 1
    assert 'no index built yet' in missing.text
    assert '0 hits' not in missing.text


def test_status_names_documents_left_behind_by_an_earlier_different_root(
        tmp_path, db):
    # This is the `roots:` line's own version of the "0 hits" defect: a build
    # naming one root never touches rows from an EARLIER build that named a
    # different one (by design -- see `build`'s docstring), so the database
    # ends up holding more than the last build's roots describe. `status`
    # must say so rather than silently reporting only the newest root list,
    # which is exactly the shape of the real incident (four PDFs indexed
    # from outside `knowledge/library` and `knowledge/MC`, `status` still
    # naming only those two).
    old_root = tmp_path / 'elsewhere'
    old_root.mkdir()
    make_pdf(old_root / 'outsider.pdf', ['a document from another root'])
    run('build', '--root', str(old_root), '--db', db)

    make_pdf(lib_dir(tmp_path) / 'insider.pdf', ['a document in the real root'])
    run('build', '--root', str(lib_dir(tmp_path)), '--db', db)

    r = run('status', '--db', db)
    assert r.code == 0
    assert one(db, 'SELECT COUNT(*) FROM docs') == 2
    assert 'OUTSIDE the roots' in r.text
    assert 'outsider.pdf' in r.text


def lib_dir(tmp_path):
    d = tmp_path / 'reallib'
    (d / 'notes').mkdir(parents=True, exist_ok=True)
    return d


def test_status_says_nothing_extra_when_everything_is_under_the_roots(
        lib, db):
    make_pdf(lib / 'guide.pdf', ['content'])
    run('build', '--root', str(lib), '--db', db)
    r = run('status', '--db', db)
    assert 'OUTSIDE the roots' not in r.text


def test_status_without_an_index_says_so_rather_than_printing_zeroes(db):
    r = run('status', '--db', db + '.nope')
    assert r.code == 1
    assert 'no index built yet' in r.text
    assert 'docs:' not in r.text


def test_a_broken_raw_query_is_a_refusal_not_zero_hits(shapes):
    r = run('search', 'AND OR', '--db', shapes, '--raw')
    assert r.code == 1
    assert 'could not run that query' in r.text
    assert '0 hits' not in r.text


# ---------------------------------------------------------------------------
# --doc filter
# ---------------------------------------------------------------------------
def test_doc_filter_accepts_a_hash_prefix_or_a_path_substring(lib, db):
    make_pdf(lib / 'alpha.pdf', ['shared word alpha'])
    make_pdf(lib / 'beta.pdf', ['shared word beta'])
    run('build', '--root', str(lib), '--db', db)

    r = run('search', 'shared', '--db', db)
    assert '2 hit(s)' in r.text

    r = run('search', 'shared', '--db', db, '--doc', 'alpha')
    assert '1 hit(s)' in r.text
    assert 'alpha.pdf' in r.text and 'beta.pdf' not in r.text

    digest = one(db, "SELECT hash FROM docs WHERE relpath LIKE '%beta.pdf'")
    r = run('search', 'shared', '--db', db, '--doc', digest[:10])
    assert '1 hit(s)' in r.text
    assert 'beta.pdf' in r.text


def test_kind_narrows_to_pages_or_notes(lib, db):
    make_pdf(lib / 'alpha.pdf', ['shared word'])
    (lib / 'notes' / 'M1-alpha.md').write_text('shared word\n',
                                               encoding='utf-8')
    run('build', '--root', str(lib), '--db', db)
    assert '2 hit(s)' in run('search', 'shared', '--db', db).text
    assert '[note]' not in run('search', 'shared', '--db', db,
                               '--kind', 'pages').text
    assert '[page]' not in run('search', 'shared', '--db', db,
                               '--kind', 'notes').text


# ---------------------------------------------------------------------------
# incremental re-ingest
# ---------------------------------------------------------------------------
def test_a_second_build_changes_nothing_and_duplicates_nothing(lib, db):
    make_pdf(lib / 'guide.pdf', ['page one', 'page two'])
    (lib / 'notes' / 'M4-guide.md').write_text('note\n', encoding='utf-8')
    run('build', '--root', str(lib), '--db', db)
    before = (one(db, 'SELECT COUNT(*) FROM docs'),
              one(db, 'SELECT COUNT(*) FROM pages'),
              one(db, 'SELECT COUNT(*) FROM notes'))

    r = run('build', '--root', str(lib), '--db', db)
    assert 'docs: 0 new, 1 unchanged, 0 moved, 0 removed' in r.text
    assert 'pages: +0, -0' in r.text
    assert 'notes: 0 new, 0 changed, 1 unchanged, 0 removed' in r.text
    after = (one(db, 'SELECT COUNT(*) FROM docs'),
             one(db, 'SELECT COUNT(*) FROM pages'),
             one(db, 'SELECT COUNT(*) FROM notes'))
    assert before == after
    # and the FTS side did not double either
    assert one(db, 'SELECT COUNT(*) FROM pages_fts') == before[1]


def test_a_moved_file_is_the_same_document(lib, db):
    make_pdf(lib / 'guide.pdf', ['page one', 'page two'])
    run('build', '--root', str(lib), '--db', db)
    digest = one(db, 'SELECT hash FROM docs')

    (lib / 'sub').mkdir()
    os.replace(str(lib / 'guide.pdf'), str(lib / 'sub' / 'guide.pdf'))
    r = run('build', '--root', str(lib), '--db', db)

    assert 'docs: 0 new, 1 unchanged, 1 moved, 0 removed' in r.text
    assert one(db, 'SELECT COUNT(*) FROM docs') == 1
    assert one(db, 'SELECT COUNT(*) FROM pages') == 2
    assert one(db, 'SELECT hash FROM docs') == digest
    assert one(db, 'SELECT relpath FROM docs').endswith('sub/guide.pdf')


def test_changed_bytes_replace_the_old_document_rather_than_joining_it(
        lib, db):
    make_pdf(lib / 'guide.pdf', ['the original wording'])
    run('build', '--root', str(lib), '--db', db)
    old = one(db, 'SELECT hash FROM docs')

    make_pdf(lib / 'guide.pdf', ['the revised wording', 'and a new page'])
    r = run('build', '--root', str(lib), '--db', db)

    assert 'docs: 1 new, 0 unchanged, 0 moved, 1 removed' in r.text
    assert one(db, 'SELECT COUNT(*) FROM docs') == 1
    assert one(db, 'SELECT COUNT(*) FROM pages') == 2
    assert one(db, 'SELECT hash FROM docs') != old
    _, p = hits(db, 'revised')
    assert p == [1]
    assert hits(db, 'original')[1] == []


def test_a_changed_note_is_reindexed_not_appended(lib, db):
    note = lib / 'notes' / 'M4-guide.md'
    note.write_text('the original claim\n', encoding='utf-8')
    run('build', '--root', str(lib), '--db', db)
    note.write_text('the corrected claim\n', encoding='utf-8')
    r = run('build', '--root', str(lib), '--db', db)

    assert 'notes: 0 new, 1 changed, 0 unchanged, 0 removed' in r.text
    assert one(db, 'SELECT COUNT(*) FROM notes') == 1
    assert '1 hit(s)' in run('search', 'corrected', '--db', db).text
    assert '0 hits' in run('search', 'original', '--db', db).text


def test_a_vanished_file_loses_its_rows_and_the_count_is_printed(lib, db):
    make_pdf(lib / 'keep.pdf', ['stays put'])
    make_pdf(lib / 'drop.pdf', ['goes away', 'second page'])
    (lib / 'notes' / 'M1-drop.md').write_text('note\n', encoding='utf-8')
    run('build', '--root', str(lib), '--db', db)

    os.remove(str(lib / 'drop.pdf'))
    os.remove(str(lib / 'notes' / 'M1-drop.md'))
    r = run('build', '--root', str(lib), '--db', db)

    assert 'docs: 0 new, 1 unchanged, 0 moved, 1 removed' in r.text
    assert 'pages: +0, -2' in r.text
    assert 'notes: 0 new, 0 changed, 0 unchanged, 1 removed' in r.text
    assert one(db, 'SELECT COUNT(*) FROM docs') == 1
    assert one(db, 'SELECT COUNT(*) FROM pages') == 1
    assert one(db, 'SELECT COUNT(*) FROM notes') == 0
    assert '0 hits' in run('search', 'goes away', '--db', db).text


def test_a_build_of_one_root_does_not_delete_another_roots_rows(tmp_path, db):
    a = tmp_path / 'a'
    b = tmp_path / 'b'
    a.mkdir()
    b.mkdir()
    make_pdf(a / 'a.pdf', ['alpha content'])
    make_pdf(b / 'b.pdf', ['bravo content'])

    run('build', '--root', str(a), '--root', str(b), '--db', db)
    assert one(db, 'SELECT COUNT(*) FROM docs') == 2

    # Re-run naming only one root. The other root's document was not looked
    # at, which is not the same as it having vanished.
    r = run('build', '--root', str(a), '--db', db)
    assert 'docs: 0 new, 1 unchanged, 0 moved, 0 removed' in r.text
    assert one(db, 'SELECT COUNT(*) FROM docs') == 2
    assert '1 hit(s)' in run('search', 'bravo', '--db', db).text


def test_two_paths_with_identical_bytes_are_one_document(lib, db):
    make_pdf(lib / 'first.pdf', ['identical content'])
    (lib / 'copy').mkdir()
    import shutil
    shutil.copyfile(str(lib / 'first.pdf'), str(lib / 'copy' / 'second.pdf'))

    r = run('build', '--root', str(lib), '--db', db)
    assert 'duplicate content, not indexed twice' in r.text
    assert one(db, 'SELECT COUNT(*) FROM docs') == 1
    assert one(db, 'SELECT COUNT(*) FROM pages') == 1


def test_a_missing_root_is_reported_and_is_not_a_reason_to_delete(lib, db):
    make_pdf(lib / 'guide.pdf', ['content'])
    run('build', '--root', str(lib), '--db', db)
    r = run('build', '--root', str(lib / 'nope'), '--db', db)
    assert 'root not found, skipped' in r.text
    assert one(db, 'SELECT COUNT(*) FROM docs') == 1


def test_rebuild_starts_from_empty(lib, db):
    make_pdf(lib / 'guide.pdf', ['content here'])
    run('build', '--root', str(lib), '--db', db)
    r = run('build', '--root', str(lib), '--db', db, '--rebuild')
    assert 'dropped every table' in r.text
    assert 'docs: 1 new' in r.text
    assert one(db, 'SELECT COUNT(*) FROM docs') == 1
    assert one(db, 'SELECT COUNT(*) FROM pages') == 1


def test_an_index_from_an_older_schema_is_wiped_rather_than_half_read(lib, db):
    make_pdf(lib / 'guide.pdf', ['content here'])
    run('build', '--root', str(lib), '--db', db)

    con = sqlite3.connect(db)
    con.execute("UPDATE meta SET value = '0' WHERE key = 'schema_version'")
    con.commit()
    con.close()

    # search refuses; it does not report 0 hits against a schema it cannot read
    r = run('search', 'content', '--db', db)
    assert r.code == 1
    assert 'another schema version' in r.text
    assert '0 hits' not in r.text

    r = run('build', '--root', str(lib), '--db', db)
    assert 'rebuilding from scratch' in r.text
    assert one(db, 'SELECT COUNT(*) FROM docs') == 1


# ---------------------------------------------------------------------------
# defaults / plumbing
# ---------------------------------------------------------------------------
def test_the_default_db_sits_inside_the_gitignored_library_folder():
    assert libindex.DEFAULT_DB.replace('\\', '/').endswith(
        'knowledge/library/index.sqlite')


def test_the_default_roots_are_the_two_document_drops():
    got = [libindex.relpath(r) for r in libindex.DEFAULT_ROOTS]
    assert got == ['knowledge/library', 'knowledge/MC']


def test_no_subcommand_prints_help_and_fails():
    assert run().code == 1


def test_search_prints_the_expression_it_ran(shapes):
    r = run('search', 'P316', '--db', shapes)
    assert 'query: "P316"' in r.text