will
8e6ee31596
ci / test (push) Successful in 6m18s
History-free snapshot of the private development repository at cbe45c5 (2026-09-09). Code, docs and public catalogue only; no plant or job knowledge layers, no client or job names (tools/scrub.py clean). The intro deck is left out of this snapshot because its slide images could not be checked for job drawings.
1328 lines
54 KiB
Python
1328 lines
54 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Tests for tools/aiextract_run.py -- the DSV4F-vision half of Design Extract.
|
|
|
|
Loaded by path, like `test_knowledgelint.py` and `test_scrub.py`: the tool
|
|
is host CPython (it needs a real HTTP client and, eventually, a real dsh)
|
|
and lives outside `WillTools.extension` entirely.
|
|
|
|
What is tested here is the part that makes an unsupervised, detached process
|
|
safe to run at all: `apply_ai_fill` (never overwrite a value the mechanical
|
|
reader placed; only fill what was empty, or add distinct new lines to the
|
|
two collections of independent entries) and `apply_failure_note` (a run that
|
|
cannot complete leaves every field exactly as it found them and says why in
|
|
one line). The dsh round trip is exercised with the HTTP boundary faked --
|
|
`transport` is a plain callable substituted for `urllib.request.urlopen`,
|
|
per `_rpc`'s own docstring -- never a real socket. The dsh wire format
|
|
itself (payload shapes, whether `session.history` really carries a status
|
|
field) is UNVERIFIED against a live dsh; see aiextract_run.py's module
|
|
docstring. That gap, and the Revit-side image export in the button, are on
|
|
the owner-must-verify list, not here.
|
|
"""
|
|
import io
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
|
|
os.path.abspath(__file__))))
|
|
TOOLS = os.path.join(ROOT, 'tools')
|
|
if TOOLS not in sys.path:
|
|
sys.path.insert(0, TOOLS)
|
|
|
|
aiextract_run = pytest.importorskip('aiextract_run')
|
|
|
|
from vibe import specformat
|
|
|
|
|
|
# --------------------------------------------------------------- fixtures
|
|
|
|
def _blank_spec(design='PNL1'):
|
|
"""A spec shaped the way `designspec.build_spec` actually leaves one --
|
|
schema set, design number set, source.extractor stamped 'regex' -- so a
|
|
test asserting the AI pass left provenance alone has a real provenance
|
|
to compare against instead of the raw template's all-None source block.
|
|
"""
|
|
spec = json.loads(json.dumps(specformat.SPEC_TEMPLATE))
|
|
spec['design_number'] = design
|
|
spec['schema'] = specformat.SCHEMA_VERSION
|
|
spec['source']['extractor'] = 'regex'
|
|
return spec
|
|
|
|
|
|
def _write_spec(path, spec):
|
|
with io.open(path, 'w', encoding='utf-8') as fh:
|
|
fh.write(specformat.spec_text(spec))
|
|
|
|
|
|
def _read_spec(path):
|
|
with io.open(path, 'r', encoding='utf-8') as fh:
|
|
return specformat.normalize_spec(specformat.parse_spec_text(fh.read()))
|
|
|
|
|
|
# ------------------------------------------------------------ apply_ai_fill
|
|
|
|
def test_never_overwrites_a_scalar_the_regex_reader_already_placed():
|
|
spec = _blank_spec()
|
|
spec['concrete']['fc_psi'] = 6000
|
|
ai = {'concrete': {'fc_psi': 9999, 'fci_psi': 3500}}
|
|
|
|
merged, filled = aiextract_run.apply_ai_fill(spec, ai)
|
|
|
|
assert merged['concrete']['fc_psi'] == 6000 # untouched
|
|
assert merged['concrete']['fci_psi'] == 3500 # filled
|
|
assert filled == ['concrete.fci_psi']
|
|
|
|
|
|
def test_fills_a_blank_scalar():
|
|
spec = _blank_spec()
|
|
ai = {'thickness_in': 8.0}
|
|
merged, filled = aiextract_run.apply_ai_fill(spec, ai)
|
|
assert merged['thickness_in'] == 8.0
|
|
assert filled == ['thickness_in']
|
|
|
|
|
|
def test_does_not_fill_when_ai_proposes_nothing():
|
|
spec = _blank_spec()
|
|
merged, filled = aiextract_run.apply_ai_fill(spec, {})
|
|
assert merged == spec
|
|
assert filled == []
|
|
|
|
|
|
def test_structured_list_filled_only_when_currently_empty():
|
|
spec = _blank_spec()
|
|
ai = {'reinforcing': {'strand_layout': {
|
|
'columns_in': [3.5, 16.5, 40.5],
|
|
'rows': [{'from_bottom_in': 1.75, 'strands': [True, True, True]}],
|
|
'total_in': 44.0,
|
|
}}}
|
|
merged, filled = aiextract_run.apply_ai_fill(spec, ai)
|
|
layout = merged['reinforcing']['strand_layout']
|
|
assert layout['columns_in'] == [3.5, 16.5, 40.5]
|
|
assert layout['total_in'] == 44.0
|
|
assert 'reinforcing.strand_layout.columns_in' in filled
|
|
assert 'reinforcing.strand_layout.rows' in filled
|
|
assert 'reinforcing.strand_layout.total_in' in filled
|
|
|
|
|
|
def test_structured_list_never_touched_once_non_empty():
|
|
spec = _blank_spec()
|
|
placed = [{'qty': 14, 'dia_in': 0.5, 'grade_ksi': 270, 'it_lb': 31600,
|
|
'role': 'main', 'note': None}]
|
|
spec['reinforcing']['strand_groups'] = placed
|
|
ai = {'reinforcing': {'strand_groups': [
|
|
{'qty': 99, 'dia_in': 0.6, 'grade_ksi': 300, 'it_lb': 1,
|
|
'role': 'bogus', 'note': None}]}}
|
|
merged, filled = aiextract_run.apply_ai_fill(spec, ai)
|
|
assert merged['reinforcing']['strand_groups'] == placed
|
|
assert filled == []
|
|
|
|
|
|
def test_datum_is_never_a_content_path_at_all():
|
|
# The datum is plant knowledge, never something the model may set --
|
|
# apply_ai_fill must not even have a rule for it under any name.
|
|
assert not any('datum' in p for p in aiextract_run.CONTENT_PATHS)
|
|
assert not any('datum' in p for p in aiextract_run.STRUCTURED_LIST_PATHS)
|
|
|
|
|
|
def test_atomic_lists_append_new_distinct_entries_and_keep_old_ones():
|
|
spec = _blank_spec()
|
|
spec['unresolved'] = [{'text': 'STRIPPING DETAIL < 25\'', 'reason': 'prose'}]
|
|
ai = {'unresolved': [
|
|
{'text': 'STRIPPING DETAIL < 25\'', 'reason': 'prose'}, # duplicate
|
|
{'text': 'chain total disagrees with piece width by 1/4"',
|
|
'reason': 'flagged by vision pass'},
|
|
]}
|
|
merged, filled = aiextract_run.apply_ai_fill(spec, ai)
|
|
assert len(merged['unresolved']) == 2
|
|
assert merged['unresolved'][0]['text'] == 'STRIPPING DETAIL < 25\''
|
|
assert merged['unresolved'][1]['text'].startswith('chain total')
|
|
assert 'unresolved' in filled
|
|
|
|
|
|
def test_atomic_list_merge_is_case_and_whitespace_tolerant_for_dedupe():
|
|
spec = _blank_spec()
|
|
spec['notes'] = ['USE MC STANDARD REINFORCEMENT FOR CUT DOWN PANELS']
|
|
ai = {'notes': ['use mc standard reinforcement for cut down panels']}
|
|
merged, filled = aiextract_run.apply_ai_fill(spec, ai)
|
|
assert len(merged['notes']) == 1
|
|
assert filled == []
|
|
|
|
|
|
def test_confidence_accepted_only_for_paths_actually_just_filled():
|
|
spec = _blank_spec()
|
|
spec['concrete']['fc_psi'] = 6000
|
|
ai = {
|
|
'concrete': {'fci_psi': 3500},
|
|
# This one names an ALREADY-FILLED field, so it must be dropped even
|
|
# though the model is "confident" about it -- it did not just fill it.
|
|
'confidence': {'concrete.fci_psi': 'high', 'concrete.fc_psi': 'high'},
|
|
}
|
|
merged, _filled = aiextract_run.apply_ai_fill(spec, ai)
|
|
assert merged['confidence'] == {'concrete.fci_psi': 'high'}
|
|
|
|
|
|
def test_apply_ai_fill_never_mutates_its_input():
|
|
spec = _blank_spec()
|
|
spec['concrete']['fc_psi'] = 6000
|
|
before = json.loads(json.dumps(spec))
|
|
aiextract_run.apply_ai_fill(spec, {'concrete': {'fci_psi': 3500}})
|
|
assert spec == before
|
|
|
|
|
|
# --------------------------------------------------------- apply_failure_note
|
|
|
|
def test_failure_note_leaves_every_field_value_untouched():
|
|
spec = _blank_spec()
|
|
spec['concrete']['fc_psi'] = 6000
|
|
spec['unresolved'] = [{'text': 'foo', 'reason': 'bar'}]
|
|
result = aiextract_run.apply_failure_note(spec, 'DshTimeout: no reply within 180s')
|
|
assert result['concrete']['fc_psi'] == 6000
|
|
assert result['source'] == spec['source'] # untouched, no stamp
|
|
assert result['unresolved'][0] == {'text': 'foo', 'reason': 'bar'}
|
|
assert len(result['unresolved']) == 2
|
|
assert result['unresolved'][1]['text'] == aiextract_run.FAILURE_REASON_PREFIX
|
|
assert 'DshTimeout' in result['unresolved'][1]['reason']
|
|
|
|
|
|
def test_failure_note_does_not_pile_up_on_repeated_identical_failures():
|
|
spec = _blank_spec()
|
|
once = aiextract_run.apply_failure_note(spec, 'same reason')
|
|
twice = aiextract_run.apply_failure_note(once, 'same reason')
|
|
assert len(twice['unresolved']) == 1
|
|
|
|
|
|
# ------------------------------------------------------- extract_json_payload
|
|
|
|
def test_extract_json_payload_reads_a_fenced_block():
|
|
reply = ('Here is what I found.\n\n'
|
|
'```json\n{"concrete": {"fci_psi": 3500}}\n```\n\n'
|
|
'Summary: filled fci_psi.')
|
|
payload = aiextract_run.extract_json_payload(reply)
|
|
assert payload == {'concrete': {'fci_psi': 3500}}
|
|
|
|
|
|
def test_extract_json_payload_falls_back_to_first_balanced_object():
|
|
reply = 'no fence here, just {"thickness_in": 8.0} inline.'
|
|
assert aiextract_run.extract_json_payload(reply) == {'thickness_in': 8.0}
|
|
|
|
|
|
def test_extract_json_payload_raises_rather_than_returning_empty():
|
|
with pytest.raises(ValueError):
|
|
aiextract_run.extract_json_payload('I could not find anything.')
|
|
|
|
|
|
def test_extract_json_payload_rejects_a_non_object_json_value():
|
|
with pytest.raises(ValueError):
|
|
aiextract_run.extract_json_payload('```json\n[1, 2, 3]\n```')
|
|
|
|
|
|
# ----------------------------------------------------------------- prompt
|
|
|
|
def test_build_prompt_includes_datum_and_evidence_and_template():
|
|
spec = _blank_spec()
|
|
prompt = aiextract_run.build_prompt(
|
|
evidence={'design_number': 'PNL1'}, regex_spec=spec,
|
|
datum_clause='the LEFT rail (datum = "left_rail").',
|
|
image_paths=[])
|
|
assert 'the LEFT rail' in prompt
|
|
assert '"design_number": "PNL1"' in prompt
|
|
assert 'strand_layout' in prompt
|
|
assert 'no drawing image was available' in prompt.lower()
|
|
|
|
|
|
def test_build_prompt_lists_image_paths_when_given():
|
|
spec = _blank_spec()
|
|
prompt = aiextract_run.build_prompt(
|
|
evidence={}, regex_spec=spec, datum_clause='d',
|
|
image_paths=['C:\\job\\PNL1.summary.png'])
|
|
assert 'C:\\job\\PNL1.summary.png' in prompt
|
|
assert 'read_image' in prompt
|
|
|
|
|
|
def test_build_prompt_strand_chain_rules_cover_concatenation_and_rows():
|
|
# rule 4a's "How to read the chain" sub-list, added after the real 10"
|
|
# slab run that gave up on one 12-segment chain against a stated 19
|
|
# strands -- the drawing actually carried two rows, each with its own
|
|
# chain, and the bottom chain arrived as four separate dimension
|
|
# elements that needed concatenating. Each assertion below is a short,
|
|
# distinctive phrase so a wording tweak elsewhere in the rule does not
|
|
# break the test.
|
|
spec = _blank_spec()
|
|
prompt = aiextract_run.build_prompt(
|
|
evidence={}, regex_spec=spec, datum_clause='d', image_paths=[])
|
|
|
|
# (a) same-y dimensions concatenate into one chain.
|
|
assert 'concatenate them left to right' in prompt
|
|
# (b) one chain per row; a full grid shares a single chain.
|
|
assert 'One chain per row' in prompt
|
|
assert 'single chain shared by every row' in prompt
|
|
# (c) a chain whose total matches neither the width nor a row is ignored.
|
|
assert 'NOT a strand chain' in prompt
|
|
# count check against the stated quantity (and symbols, when present).
|
|
assert 'MUST equal the stated quantity' in prompt
|
|
# the STRAND PATTERN (matrix) unresolved-reason pointer.
|
|
assert 'STRAND PATTERN (matrix)' in prompt
|
|
|
|
|
|
def _leaf_paths(node, prefix=''):
|
|
paths = []
|
|
for key, value in node.items():
|
|
path = '{0}.{1}'.format(prefix, key) if prefix else key
|
|
if isinstance(value, dict) and value:
|
|
paths.extend(_leaf_paths(value, path))
|
|
else:
|
|
paths.append(path)
|
|
return paths
|
|
|
|
|
|
def test_reply_template_matches_the_merge_rule_tables_exactly():
|
|
# The old opus-CLI prompt drifted from the spec's real field names
|
|
# (`spacings_in` / `positions_in` / `strands_per_row`, which stopped
|
|
# existing when the matrix shape landed) because nothing exercised it.
|
|
# This pins AI_REPLY_TEMPLATE -- what the prompt actually asks the model
|
|
# to send back -- against the three tables apply_ai_fill actually reads.
|
|
leaves = set(_leaf_paths(aiextract_run.AI_REPLY_TEMPLATE))
|
|
expected = (set(aiextract_run.CONTENT_PATHS) |
|
|
set(aiextract_run.ATOMIC_LIST_PATHS) | {'confidence'})
|
|
assert leaves == expected
|
|
|
|
|
|
# -------------------------------------------------------------- resolve_model
|
|
#
|
|
# `resolve_model()` now returns `(model, provider, why)` -- see the module
|
|
# docstring's point 4: `session.selectModel` needs an explicit provider id
|
|
# alongside the model id, and this file was rewritten to fail closed on
|
|
# either rather than guess. Every test here stubs `_find_provider_for_model_id`
|
|
# unless it is specifically testing provider resolution, so the model-only
|
|
# assertions above stay isolated from the real ~/.dsh/settings.yaml this
|
|
# machine happens to have -- the same isolation `_scan_settings_for_vision_model`
|
|
# is already stubbed for below.
|
|
|
|
def test_resolve_model_prefers_an_explicit_job_setting(monkeypatch):
|
|
monkeypatch.delenv(aiextract_run.MODEL_ENV, raising=False)
|
|
monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False)
|
|
monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id',
|
|
lambda *_a, **_k: ('stub-provider', None))
|
|
model, provider, why = aiextract_run.resolve_model(
|
|
{'model': 'named-explicitly'})
|
|
assert model == 'named-explicitly'
|
|
assert provider == 'stub-provider'
|
|
assert why is None
|
|
|
|
|
|
def test_resolve_model_falls_back_to_the_env_var(monkeypatch):
|
|
monkeypatch.setenv(aiextract_run.MODEL_ENV, 'from-env')
|
|
monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False)
|
|
monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id',
|
|
lambda *_a, **_k: ('stub-provider', None))
|
|
model, provider, why = aiextract_run.resolve_model({})
|
|
assert model == 'from-env'
|
|
assert provider == 'stub-provider'
|
|
assert why is None
|
|
|
|
|
|
def test_resolve_model_uses_settings_scan_result(monkeypatch):
|
|
monkeypatch.delenv(aiextract_run.MODEL_ENV, raising=False)
|
|
monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False)
|
|
monkeypatch.setattr(aiextract_run, '_scan_settings_for_vision_model',
|
|
lambda: ('commandcode-vision', 'commandcode', None))
|
|
model, provider, why = aiextract_run.resolve_model({})
|
|
assert model == 'commandcode-vision'
|
|
# The scan already knew which provider hosts it -- free, no separate
|
|
# `_find_provider_for_model_id` lookup needed.
|
|
assert provider == 'commandcode'
|
|
assert why is None
|
|
|
|
|
|
def test_resolve_model_falls_back_to_unconfirmed_constant_and_says_so(monkeypatch):
|
|
monkeypatch.delenv(aiextract_run.MODEL_ENV, raising=False)
|
|
monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False)
|
|
monkeypatch.setattr(aiextract_run, '_scan_settings_for_vision_model',
|
|
lambda: (None, None, 'no settings file'))
|
|
monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id',
|
|
lambda *_a, **_k: (None, 'no settings file'))
|
|
model, provider, why = aiextract_run.resolve_model({})
|
|
assert model == aiextract_run.FALLBACK_VISION_MODEL
|
|
assert provider is None
|
|
assert why is not None and 'unconfirmed' in why
|
|
|
|
|
|
# --------------------------------------------------- resolve_model: provider
|
|
|
|
def test_resolve_model_prefers_an_explicit_job_provider(monkeypatch):
|
|
monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False)
|
|
|
|
def _boom(*_a, **_k):
|
|
raise AssertionError('should not need to look the provider up')
|
|
|
|
monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id', _boom)
|
|
model, provider, why = aiextract_run.resolve_model(
|
|
{'model': 'named-explicitly', 'provider': 'named-provider'})
|
|
assert model == 'named-explicitly'
|
|
assert provider == 'named-provider'
|
|
assert why is None
|
|
|
|
|
|
def test_resolve_model_falls_back_to_the_provider_env_var(monkeypatch):
|
|
monkeypatch.setenv(aiextract_run.PROVIDER_ENV, 'env-provider')
|
|
|
|
def _boom(*_a, **_k):
|
|
raise AssertionError('should not need to look the provider up')
|
|
|
|
monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id', _boom)
|
|
model, provider, why = aiextract_run.resolve_model(
|
|
{'model': 'named-explicitly'})
|
|
assert provider == 'env-provider'
|
|
assert why is None
|
|
|
|
|
|
def test_resolve_model_refuses_to_guess_a_provider_it_cannot_find(monkeypatch):
|
|
# THE FAIL-CLOSED CASE: a model id came from somewhere, but nothing
|
|
# (job, env, settings.yaml) can say which provider serves it --
|
|
# `session.selectModel` requires one, so this must never invent it.
|
|
monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False)
|
|
monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id',
|
|
lambda *_a, **_k: (None, 'no entry for this model'))
|
|
model, provider, why = aiextract_run.resolve_model(
|
|
{'model': 'named-explicitly'})
|
|
assert model == 'named-explicitly'
|
|
assert provider is None
|
|
assert why is not None and 'provider' in why.lower()
|
|
|
|
|
|
# The REAL shape, mimicked. Coordinator, 2026-09-01, quoting the actual file
|
|
# on the machine this ships to (`~/.dsh/settings.yaml`, line 407ff): an `id:`
|
|
# list item and its `input:` list are SEVERAL LINES APART, separated by other
|
|
# keys, inside a deeply nested `providers: [...] -> models: [...]` structure
|
|
# -- not the flat `mapping-key: {input: [...]}` shape the old line-scan
|
|
# heuristic (and the tests that used to live here) assumed. This fixture is
|
|
# that shape, and it is what the parser is actually judged against now.
|
|
_REAL_SETTINGS_FIXTURE = (
|
|
'version: 1\n'
|
|
'providers:\n'
|
|
' - name: commandcode\n'
|
|
' base_url: https://example.invalid\n'
|
|
' models:\n'
|
|
' - id: deepseek/deepseek-v4-flash\n'
|
|
' display_name: DeepSeek V4 Flash\n'
|
|
' context_window: 128000\n'
|
|
' pricing:\n'
|
|
' input_per_mtok: 0.1\n'
|
|
' output_per_mtok: 0.3\n'
|
|
' input:\n'
|
|
' - text\n'
|
|
' - id: deepseek/deepseek-v4-flash-vision-exp\n'
|
|
' display_name: DeepSeek V4 Flash Vision (Experimental)\n'
|
|
' context_window: 128000\n'
|
|
' pricing:\n'
|
|
' input_per_mtok: 0.15\n'
|
|
' output_per_mtok: 0.4\n'
|
|
' off_peak_discount: 0.5\n'
|
|
' capabilities:\n'
|
|
' - vision\n'
|
|
' - tool_use\n'
|
|
' input:\n'
|
|
' - text\n'
|
|
' - image\n'
|
|
' output:\n'
|
|
' - text\n'
|
|
)
|
|
|
|
|
|
def test_scan_settings_finds_the_real_deeply_nested_entry(tmp_path):
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text(_REAL_SETTINGS_FIXTURE, encoding='utf-8')
|
|
model, provider, why = aiextract_run._scan_settings_for_vision_model(
|
|
str(settings))
|
|
assert model == 'deepseek/deepseek-v4-flash-vision-exp'
|
|
assert provider == 'commandcode' # from the provider's `name:`
|
|
assert why is None
|
|
|
|
|
|
def test_resolve_model_finds_the_real_entry_end_to_end(tmp_path, monkeypatch):
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text(_REAL_SETTINGS_FIXTURE, encoding='utf-8')
|
|
monkeypatch.delenv(aiextract_run.MODEL_ENV, raising=False)
|
|
original_scan = aiextract_run._scan_settings_for_vision_model
|
|
monkeypatch.setattr(aiextract_run, '_scan_settings_for_vision_model',
|
|
lambda: original_scan(str(settings)))
|
|
model, provider, why = aiextract_run.resolve_model({})
|
|
assert model == 'deepseek/deepseek-v4-flash-vision-exp'
|
|
assert provider == 'commandcode'
|
|
assert why is None
|
|
|
|
|
|
# The OTHER real shape. Confirmed live, 2026-09-03, reading this machine's
|
|
# actual `~/.dsh/settings.yaml`: `providers` there is not a list at all --
|
|
# it is a MAPPING keyed by provider id (`llm-pi-ai: providers: {commandcode:
|
|
# {models: [...]}, opencode: {...}}`), and `commandcode` (the dict key) is
|
|
# exactly the string `session.selectModel` needs as `provider`. Both shapes
|
|
# have to resolve a provider id; `_iter_model_entries` is what tolerates
|
|
# either (see its docstring).
|
|
_REAL_DICT_KEYED_PROVIDERS_FIXTURE = (
|
|
'llm-pi-ai:\n'
|
|
' providers:\n'
|
|
' commandcode:\n'
|
|
' displayName: Command Code API\n'
|
|
' baseURL: https://api.commandcode.ai/provider/v1\n'
|
|
' models:\n'
|
|
' - id: deepseek/deepseek-v4-flash\n'
|
|
' name: DeepSeek V4 Flash (latest)\n'
|
|
' - id: deepseek/deepseek-v4-flash-vision-exp\n'
|
|
' name: DeepSeek V4 Flash Vision (exp)\n'
|
|
' input:\n'
|
|
' - text\n'
|
|
' - image\n'
|
|
)
|
|
|
|
|
|
def test_scan_settings_finds_the_provider_id_when_providers_is_a_dict(tmp_path):
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text(_REAL_DICT_KEYED_PROVIDERS_FIXTURE, encoding='utf-8')
|
|
model, provider, why = aiextract_run._scan_settings_for_vision_model(
|
|
str(settings))
|
|
assert model == 'deepseek/deepseek-v4-flash-vision-exp'
|
|
assert provider == 'commandcode' # the dict KEY, not a `name:`
|
|
assert why is None
|
|
|
|
|
|
def test_scan_settings_picks_the_vision_named_id_when_several_qualify(tmp_path):
|
|
# Two entries both declare image input; only one's id says "vision" --
|
|
# that one wins without asking, per the owner's own naming convention.
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text(
|
|
'providers:\n'
|
|
' - models:\n'
|
|
' - id: some/multimodal-model\n'
|
|
' input: [text, image]\n'
|
|
' - id: some/vision-model\n'
|
|
' input: [text, image]\n',
|
|
encoding='utf-8')
|
|
model, _provider, why = aiextract_run._scan_settings_for_vision_model(
|
|
str(settings))
|
|
assert model == 'some/vision-model'
|
|
assert why is None
|
|
|
|
|
|
def test_scan_settings_refuses_to_guess_between_two_candidates(tmp_path):
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text(
|
|
'providers:\n'
|
|
' - models:\n'
|
|
' - id: vendor/vision-a\n'
|
|
' input: [text, image]\n'
|
|
' - id: vendor/vision-b\n'
|
|
' input: [text, image]\n',
|
|
encoding='utf-8')
|
|
model, provider, why = aiextract_run._scan_settings_for_vision_model(
|
|
str(settings))
|
|
assert model is None
|
|
assert provider is None
|
|
assert 'vendor/vision-a' in why and 'vendor/vision-b' in why
|
|
|
|
|
|
def test_scan_settings_missing_file_says_so(tmp_path):
|
|
model, provider, why = aiextract_run._scan_settings_for_vision_model(
|
|
str(tmp_path / 'does-not-exist.yaml'))
|
|
assert model is None
|
|
assert provider is None
|
|
assert 'no settings file' in why
|
|
|
|
|
|
def test_scan_settings_ignores_entries_with_id_but_no_image_input(tmp_path):
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text(_REAL_SETTINGS_FIXTURE, encoding='utf-8')
|
|
model, _provider, _why = aiextract_run._scan_settings_for_vision_model(
|
|
str(settings))
|
|
assert model != 'deepseek/deepseek-v4-flash' # the text-only sibling
|
|
|
|
|
|
def test_scan_settings_malformed_yaml_says_so_not_raises(tmp_path):
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text('providers: [this is not: valid: yaml: at all\n',
|
|
encoding='utf-8')
|
|
model, provider, why = aiextract_run._scan_settings_for_vision_model(
|
|
str(settings))
|
|
assert model is None
|
|
assert provider is None
|
|
assert 'could not parse' in why
|
|
|
|
|
|
# --------------------------------------------------- _find_provider_for_model_id
|
|
|
|
def test_find_provider_for_model_id_dict_keyed_shape(tmp_path):
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text(_REAL_DICT_KEYED_PROVIDERS_FIXTURE, encoding='utf-8')
|
|
provider, why = aiextract_run._find_provider_for_model_id(
|
|
'deepseek/deepseek-v4-flash', str(settings))
|
|
assert provider == 'commandcode'
|
|
assert why is None
|
|
|
|
|
|
def test_find_provider_for_model_id_ignores_whether_it_declares_image_input(
|
|
tmp_path):
|
|
# Unlike `_scan_settings_for_vision_model`, this does not care about
|
|
# vision -- it is asked about a model the CALLER already chose (an
|
|
# explicit job/env override), and just needs to know who serves it.
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text(_REAL_DICT_KEYED_PROVIDERS_FIXTURE, encoding='utf-8')
|
|
provider, why = aiextract_run._find_provider_for_model_id(
|
|
'deepseek/deepseek-v4-flash-vision-exp', str(settings))
|
|
assert provider == 'commandcode'
|
|
assert why is None
|
|
|
|
|
|
def test_find_provider_for_model_id_not_found_says_so(tmp_path):
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text(_REAL_DICT_KEYED_PROVIDERS_FIXTURE, encoding='utf-8')
|
|
provider, why = aiextract_run._find_provider_for_model_id(
|
|
'no/such-model', str(settings))
|
|
assert provider is None
|
|
assert 'no/such-model' in why
|
|
|
|
|
|
def test_find_provider_for_model_id_missing_file_says_so(tmp_path):
|
|
provider, why = aiextract_run._find_provider_for_model_id(
|
|
'anything', str(tmp_path / 'does-not-exist.yaml'))
|
|
assert provider is None
|
|
assert 'no settings file' in why
|
|
|
|
|
|
def test_find_provider_for_model_id_refuses_to_guess_between_two_providers(
|
|
tmp_path):
|
|
settings = tmp_path / 'settings.yaml'
|
|
settings.write_text(
|
|
'providers:\n'
|
|
' vendor-a:\n'
|
|
' models:\n'
|
|
' - id: shared/model-id\n'
|
|
' vendor-b:\n'
|
|
' models:\n'
|
|
' - id: shared/model-id\n',
|
|
encoding='utf-8')
|
|
provider, why = aiextract_run._find_provider_for_model_id(
|
|
'shared/model-id', str(settings))
|
|
assert provider is None
|
|
assert 'vendor-a' in why and 'vendor-b' in why
|
|
|
|
|
|
# --------------------------------------------------------------------- _rpc_value
|
|
#
|
|
# The bug: a real WPA5/sample_job run got `DshError: session.create did not
|
|
# return a session id` on a call that had, in fact, fully succeeded --
|
|
# `create_session` was reading `result.sessionId` when the real reply nests
|
|
# the id two deep, at `result.value.sessionId`, and the session it could not
|
|
# see was left orphaned, live, on the dsh server. This section pins the
|
|
# EXACT captured response (the regression) and every malformed variant the
|
|
# task called out, each naming the layer that failed rather than crashing.
|
|
|
|
_REAL_SESSION_CREATE_RESPONSE = {
|
|
'type': 'server-response',
|
|
'rpcId': '3d7ad657a9b4a83a5234cfd5ce185c7',
|
|
'result': {'ok': True, 'value': {
|
|
'sessionId': 'session-5969e01e-abe1-4182-a7a3-6c2f594c5419',
|
|
'agentPreset': 'standard'}},
|
|
}
|
|
|
|
|
|
def test_rpc_value_parses_the_exact_captured_session_create_response():
|
|
# THE REGRESSION. This is the literal response from the failed WPA5 run
|
|
# (owner report, 2026-09-01), byte-for-byte down to the rpcId.
|
|
value = aiextract_run._rpc_value(_REAL_SESSION_CREATE_RESPONSE,
|
|
'session.create')
|
|
assert value == {
|
|
'sessionId': 'session-5969e01e-abe1-4182-a7a3-6c2f594c5419',
|
|
'agentPreset': 'standard'}
|
|
|
|
|
|
def test_create_session_parses_the_exact_captured_response():
|
|
# Same 'result' as the real captured reply, echoing whatever rpcId this
|
|
# call actually sends (create_session itself has no control over the
|
|
# rpcId `_rpc` generates, so the transport must echo it, same as every
|
|
# other fake transport in this file -- only the 'result' body below is
|
|
# the real, captured one).
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'],
|
|
'result': _REAL_SESSION_CREATE_RESPONSE['result']}
|
|
|
|
session_id = aiextract_run.create_session('http://fake', 30, transport)
|
|
assert session_id == 'session-5969e01e-abe1-4182-a7a3-6c2f594c5419'
|
|
|
|
|
|
def test_rpc_value_refuses_when_result_ok_is_false():
|
|
res = {'rpcId': 'x', 'result': {'ok': False, 'value': {'sessionId': 'a'}}}
|
|
with pytest.raises(aiextract_run.DshError) as exc:
|
|
aiextract_run._rpc_value(res, 'session.create')
|
|
assert 'result.ok' in str(exc.value)
|
|
|
|
|
|
def test_rpc_value_refuses_when_value_is_missing():
|
|
res = {'rpcId': 'x', 'result': {'ok': True}}
|
|
with pytest.raises(aiextract_run.DshError) as exc:
|
|
aiextract_run._rpc_value(res, 'session.create')
|
|
assert 'result.value' in str(exc.value)
|
|
|
|
|
|
def test_rpc_value_refuses_when_value_is_not_an_object():
|
|
res = {'rpcId': 'x', 'result': {'ok': True, 'value': 'not-an-object'}}
|
|
with pytest.raises(aiextract_run.DshError) as exc:
|
|
aiextract_run._rpc_value(res, 'session.create')
|
|
assert 'result.value' in str(exc.value)
|
|
|
|
|
|
def test_rpc_value_refuses_when_result_is_missing_entirely():
|
|
res = {'rpcId': 'x'}
|
|
with pytest.raises(aiextract_run.DshError) as exc:
|
|
aiextract_run._rpc_value(res, 'session.create')
|
|
assert '"result"' in str(exc.value)
|
|
|
|
|
|
def test_create_session_refuses_clearly_on_the_old_flat_wrong_shape():
|
|
# The shape the FIRST version of this file (wrongly) expected --
|
|
# result.sessionId, one level, no 'ok'/'value' at all. Must refuse
|
|
# clearly (naming which check failed), never crash with a raw KeyError
|
|
# or silently return something wrong.
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'], 'result': {'sessionId': 'sess-1'}}
|
|
|
|
with pytest.raises(aiextract_run.DshError) as exc:
|
|
aiextract_run.create_session('http://fake', 30, transport)
|
|
assert 'result.ok' in str(exc.value)
|
|
|
|
|
|
def test_create_session_names_sessionid_when_value_has_no_usable_one():
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'],
|
|
'result': {'ok': True, 'value': {'agentPreset': 'standard'}}}
|
|
|
|
with pytest.raises(aiextract_run.DshError) as exc:
|
|
aiextract_run.create_session('http://fake', 30, transport)
|
|
assert 'sessionId' in str(exc.value)
|
|
|
|
|
|
def test_create_session_refuses_an_empty_string_sessionid():
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'],
|
|
'result': {'ok': True, 'value': {'sessionId': ' '}}}
|
|
|
|
with pytest.raises(aiextract_run.DshError) as exc:
|
|
aiextract_run.create_session('http://fake', 30, transport)
|
|
assert 'sessionId' in str(exc.value)
|
|
|
|
|
|
# ----------------------------------------------------------- cancel_session
|
|
|
|
def test_cancel_session_returns_true_when_the_rpc_succeeds():
|
|
calls = []
|
|
|
|
def transport(_url, body, _timeout):
|
|
calls.append(body)
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {}}}
|
|
|
|
assert aiextract_run.cancel_session('http://fake', 'sess-1', 30,
|
|
transport) is True
|
|
assert calls[0]['method'] == 'session.cancel'
|
|
assert calls[0]['payload'] == {'sessionId': 'sess-1'}
|
|
|
|
|
|
def test_cancel_session_swallows_errors_and_returns_false():
|
|
def transport(_url, _body, _timeout):
|
|
raise aiextract_run.DshTimeout('no reply')
|
|
|
|
assert aiextract_run.cancel_session('http://fake', 'sess-1', 30,
|
|
transport) is False
|
|
|
|
|
|
def test_run_cancels_the_session_when_something_fails_after_it_was_created(
|
|
tmp_path):
|
|
# The residual orphan-session risk: create_session succeeds and returns
|
|
# a real, trusted id, but something LATER in the same pass fails (here,
|
|
# the reply never settles). `run()` must attempt a best-effort
|
|
# session.cancel with that id before writing the failure note.
|
|
spec = _blank_spec()
|
|
job = _job(tmp_path, spec)
|
|
cancel_calls = []
|
|
|
|
def transport(_url, body, _timeout):
|
|
method = body['method']
|
|
rpc_id = body['rpcId']
|
|
if method == 'session.create':
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value':
|
|
{'sessionId': 'sess-orphan-check'}}}
|
|
if method == 'session.selectModel':
|
|
payload = body['payload']
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value':
|
|
{'selected': {'provider': payload['provider'],
|
|
'model': payload['model']}}}}
|
|
if method == 'session.history':
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value':
|
|
{'messages': []}}}
|
|
if method == 'session.prompt':
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {}}}
|
|
if method == 'session.cancel':
|
|
cancel_calls.append(body['payload'])
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {}}}
|
|
raise AssertionError('unexpected method: ' + method)
|
|
|
|
# A real fake clock, not real wall time: `job.get('timeout_s') or
|
|
# DEFAULT_TIMEOUT_S` means 0 would silently fall back to the 180s
|
|
# default (falsy-0 is the same trap as everywhere else `or` guards a
|
|
# possibly-zero setting), so this uses a small positive timeout plus a
|
|
# stepped fake clock/sleep -- the same pattern `_FakeClock` exists for --
|
|
# rather than actually waiting.
|
|
clock = _FakeClock()
|
|
job['timeout_s'] = 5
|
|
result = aiextract_run.run(job, transport=transport, sleep=clock.sleep,
|
|
clock=clock.time)
|
|
|
|
assert result['ok'] is False
|
|
assert cancel_calls == [{'sessionId': 'sess-orphan-check'}]
|
|
|
|
|
|
def test_run_never_half_merges_on_the_exact_captured_session_create_shape(
|
|
tmp_path):
|
|
# THE NEVER-HALF-MERGE GUARANTEE, confirmed on THIS specific failure
|
|
# mode: a session.create reply shaped exactly like the real WPA5 capture
|
|
# (see _REAL_SESSION_CREATE_RESPONSE above) but with a corrupted 'ok'
|
|
# flag, so it still fails the same way the original bug would have if
|
|
# the fix regressed. The regex-written spec's field values, and
|
|
# source.extractor, must come through byte-identical -- apply_failure_note's
|
|
# contract from yesterday, re-confirmed against today's failure mode.
|
|
spec = _blank_spec()
|
|
spec['concrete']['fc_psi'] = 6000
|
|
spec['thickness_in'] = 8.0
|
|
job = _job(tmp_path, spec)
|
|
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'],
|
|
'result': {'ok': False, 'value': {
|
|
'sessionId': 'session-5969e01e-abe1-4182-a7a3-6c2f594c5419',
|
|
'agentPreset': 'standard'}}}
|
|
|
|
result = aiextract_run.run(job, transport=transport, sleep=lambda _s: None)
|
|
|
|
assert result['ok'] is False
|
|
written = _read_spec(job['spec_path'])
|
|
assert written['concrete']['fc_psi'] == 6000
|
|
assert written['thickness_in'] == 8.0
|
|
assert written['source']['extractor'] == 'regex' # never restamped
|
|
assert 'ai_filled' not in written['source'] # never even added
|
|
assert len(written['unresolved']) == 1
|
|
assert written['unresolved'][0]['text'] == aiextract_run.FAILURE_REASON_PREFIX
|
|
assert 'result.ok' in written['unresolved'][0]['reason']
|
|
|
|
|
|
def test_run_does_not_attempt_a_cancel_when_session_create_itself_failed(
|
|
tmp_path):
|
|
# No session id was ever obtained, so there is nothing safe to cancel --
|
|
# attempting one here would mean guessing an id, which cancel_session's
|
|
# own docstring says this script must not do.
|
|
spec = _blank_spec()
|
|
job = _job(tmp_path, spec)
|
|
calls = []
|
|
|
|
def transport(_url, body, _timeout):
|
|
calls.append(body['method'])
|
|
raise aiextract_run.DshTimeout('dsh unreachable')
|
|
|
|
result = aiextract_run.run(job, transport=transport, sleep=lambda _s: None)
|
|
|
|
assert result['ok'] is False
|
|
assert 'session.cancel' not in calls
|
|
|
|
|
|
# --------------------------------------------------------------- send_prompt
|
|
|
|
def test_send_prompt_sends_the_real_measured_payload_shape():
|
|
# MEASURED live against dsh, 2026-09-03: this is the exact shape that
|
|
# was missing before -- the old code sent {'sessionId', 'text'} and dsh
|
|
# answered 'invalid payload for session.prompt' on every run.
|
|
calls = []
|
|
|
|
def transport(_url, body, _timeout):
|
|
calls.append(body)
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {}}}
|
|
|
|
aiextract_run.send_prompt('http://fake', 'sess-1', 'hello model', 30,
|
|
transport)
|
|
|
|
assert calls[0]['method'] == 'session.prompt'
|
|
assert calls[0]['payload'] == {
|
|
'sessionId': 'sess-1',
|
|
'mode': 'queue',
|
|
'content': [{'type': 'text', 'text': 'hello model'}],
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------------- select_model
|
|
#
|
|
# THE ACTUAL FIX: `session.create`'s real payload schema
|
|
# (`sessionCreateRequestSchema`, dsh-host-apiproxy) has no `model` field at
|
|
# all -- `create_session` no longer sends one (see its own docstring and the
|
|
# `test_create_session_sends_no_model_field_at_all` test below).
|
|
# `session.selectModel` is the separate RPC that actually pins a session's
|
|
# model, and `provider` is a required field of its payload.
|
|
|
|
def test_create_session_sends_no_model_field_at_all():
|
|
calls = []
|
|
|
|
def transport(_url, body, _timeout):
|
|
calls.append(body)
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'sessionId': 'sess-1', 'agentPreset': 'standard'}}}
|
|
|
|
aiextract_run.create_session('http://fake', 30, transport)
|
|
|
|
assert calls[0]['method'] == 'session.create'
|
|
assert 'model' not in calls[0]['payload']
|
|
|
|
|
|
def test_select_model_sends_the_measured_payload_shape():
|
|
# MEASURED live against dsh, 2026-09-03: session.create -> session.
|
|
# selectModel({provider: "commandcode", model:
|
|
# "deepseek/deepseek-v4-flash-vision-exp"}) -> session.models
|
|
# immediately echoed that exact pair back as `current`.
|
|
calls = []
|
|
|
|
def transport(_url, body, _timeout):
|
|
calls.append(body)
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'selected': {'provider': 'commandcode',
|
|
'model': 'deepseek/deepseek-v4-flash-vision-exp'}}}}
|
|
|
|
value = aiextract_run.select_model(
|
|
'http://fake', 'sess-1', 'commandcode',
|
|
'deepseek/deepseek-v4-flash-vision-exp', 30, transport)
|
|
|
|
assert calls[0]['method'] == 'session.selectModel'
|
|
assert calls[0]['payload'] == {
|
|
'sessionId': 'sess-1',
|
|
'provider': 'commandcode',
|
|
'model': 'deepseek/deepseek-v4-flash-vision-exp',
|
|
}
|
|
assert value['selected']['provider'] == 'commandcode'
|
|
assert value['selected']['model'] == \
|
|
'deepseek/deepseek-v4-flash-vision-exp'
|
|
|
|
|
|
# ---------------------------------------------------------- verify_session_model
|
|
#
|
|
# THE CHECK THIS TASK WAS OPENED TO ADD: neither `session.create`'s response
|
|
# nor `session.selectModel`'s own response is proof a turn actually ran on
|
|
# the requested model -- both only echo what was asked for. The
|
|
# `request/context` history event is what says what a turn ACTUALLY used.
|
|
|
|
def test_verify_session_model_passes_when_context_matches():
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'events': [_context_event(
|
|
'commandcode', 'deepseek/deepseek-v4-flash-vision-exp')],
|
|
'hasMore': False}}}
|
|
|
|
provider, model = aiextract_run.verify_session_model(
|
|
'http://fake', 'sess-1', 'commandcode',
|
|
'deepseek/deepseek-v4-flash-vision-exp', 30, transport)
|
|
|
|
assert provider == 'commandcode'
|
|
assert model == 'deepseek/deepseek-v4-flash-vision-exp'
|
|
|
|
|
|
def test_verify_session_model_raises_naming_both_on_mismatch():
|
|
# THE BUG THIS CHECK EXISTS FOR: a session can accept selectModel and
|
|
# still actually run on the account's default model -- this is the one
|
|
# place that would have caught the original failure (a vision request
|
|
# silently served by deepseek-v4-flash, no image input).
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'events': [_context_event(
|
|
'commandcode', 'deepseek/deepseek-v4-flash')],
|
|
'hasMore': False}}}
|
|
|
|
with pytest.raises(aiextract_run.DshError) as exc:
|
|
aiextract_run.verify_session_model(
|
|
'http://fake', 'sess-1', 'commandcode',
|
|
'deepseek/deepseek-v4-flash-vision-exp', 30, transport)
|
|
|
|
message = str(exc.value)
|
|
assert 'deepseek/deepseek-v4-flash-vision-exp' in message # requested
|
|
assert 'deepseek/deepseek-v4-flash' in message # actual
|
|
assert 'commandcode' in message
|
|
|
|
|
|
def test_verify_session_model_raises_when_no_context_event_seen_yet():
|
|
# "could not tell" must never pass as "must be fine" -- same discipline
|
|
# as everywhere else in this codebase (CLAUDE.md's `knowledge.same_face()`
|
|
# example).
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'events': [], 'hasMore': False}}}
|
|
|
|
with pytest.raises(aiextract_run.DshError) as exc:
|
|
aiextract_run.verify_session_model(
|
|
'http://fake', 'sess-1', 'commandcode', 'some-model', 30,
|
|
transport)
|
|
assert 'request/context' in str(exc.value)
|
|
|
|
|
|
def test_verify_session_model_uses_the_last_context_event_when_several():
|
|
# A turn can carry more than one request/context event (compaction, a
|
|
# retry); the LAST one is what the turn actually settled on.
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'events': [
|
|
_context_event('commandcode', 'deepseek/deepseek-v4-flash'),
|
|
_context_event('commandcode',
|
|
'deepseek/deepseek-v4-flash-vision-exp'),
|
|
],
|
|
'hasMore': False}}}
|
|
|
|
provider, model = aiextract_run.verify_session_model(
|
|
'http://fake', 'sess-1', 'commandcode',
|
|
'deepseek/deepseek-v4-flash-vision-exp', 30, transport)
|
|
|
|
assert provider == 'commandcode'
|
|
assert model == 'deepseek/deepseek-v4-flash-vision-exp'
|
|
|
|
|
|
def test_extract_context_events_ignores_events_of_other_types():
|
|
value = {'events': [
|
|
{'event': {'type': 'turn/start', 'data': {}}},
|
|
{'event': {'type': 'request/context', 'data': {
|
|
'provider': 'commandcode', 'model': 'm-1'}}},
|
|
{'event': {'type': 'request/header', 'data': {
|
|
'header': {'config': {'provider': 'commandcode',
|
|
'model': 'm-1'}}}}},
|
|
]}
|
|
events = aiextract_run._extract_context_events(value)
|
|
assert events == [{'provider': 'commandcode', 'model': 'm-1'}]
|
|
|
|
|
|
# ----------------------------------------------------------- _history_messages
|
|
|
|
def test_history_messages_projects_the_real_events_shape():
|
|
# user/message keeps its text; assistant/message keeps only its "text"
|
|
# blocks and drops "reasoning" ones; assistant/chunk (a streaming delta)
|
|
# is ignored entirely, same as every other event type not named here.
|
|
value = {
|
|
'events': [
|
|
{'event': {'type': 'turn/start', 'data': {}}},
|
|
{'event': {'type': 'user/message', 'data': {
|
|
'content': [{'type': 'text', 'text': 'please read this'}]}}},
|
|
{'event': {'type': 'assistant/chunk', 'data': {
|
|
'chunk': {'type': 'text-delta', 'text': 'partial...'}}}},
|
|
{'event': {'type': 'assistant/message', 'data': {
|
|
'turn': 1, 'step': 1, 'message': {
|
|
'role': 'assistant',
|
|
'content': [
|
|
{'type': 'reasoning', 'text': 'thinking it through'},
|
|
{'type': 'text', 'text': 'the settled answer'},
|
|
]}}}},
|
|
],
|
|
'hasMore': False,
|
|
'projections': {},
|
|
}
|
|
|
|
messages = aiextract_run._history_messages(value)
|
|
|
|
assert messages == [
|
|
{'role': 'user', 'text': 'please read this'},
|
|
{'role': 'assistant', 'text': 'the settled answer'},
|
|
]
|
|
|
|
|
|
def test_history_messages_still_tolerates_the_old_guessed_shape():
|
|
value = {'messages': [{'role': 'assistant', 'content': 'old-shape text'}]}
|
|
messages = aiextract_run._history_messages(value)
|
|
assert messages == [{'role': 'assistant', 'text': 'old-shape text'}]
|
|
|
|
|
|
# ------------------------------------------------------------- session_running
|
|
|
|
def test_session_running_true_when_the_session_is_listed_and_running():
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'items': [{'sessionId': 'sess-1', 'running': True}]}}}
|
|
|
|
assert aiextract_run.session_running(
|
|
'http://fake', 'sess-1', 30, transport) is True
|
|
|
|
|
|
def test_session_running_false_once_the_turn_has_finished():
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'items': [{'sessionId': 'sess-1', 'running': False}]}}}
|
|
|
|
assert aiextract_run.session_running(
|
|
'http://fake', 'sess-1', 30, transport) is False
|
|
|
|
|
|
def test_session_running_none_when_the_session_is_not_listed():
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'items': [{'sessionId': 'some-other-session', 'running': True}]}}}
|
|
|
|
assert aiextract_run.session_running(
|
|
'http://fake', 'sess-1', 30, transport) is None
|
|
|
|
|
|
# --------------------------------------------------------------- wait_for_reply
|
|
|
|
class _FakeClock(object):
|
|
def __init__(self, start=0.0, step=2.0):
|
|
self.now = start
|
|
self.step = step
|
|
|
|
def time(self):
|
|
return self.now
|
|
|
|
def sleep(self, _interval):
|
|
self.now += self.step
|
|
|
|
|
|
def test_wait_for_reply_times_out_when_nothing_ever_arrives():
|
|
clock = _FakeClock()
|
|
|
|
def transport(_url, body, _timeout):
|
|
return {'rpcId': body['rpcId'],
|
|
'result': {'ok': True, 'value': {'messages': []}}}
|
|
|
|
with pytest.raises(aiextract_run.DshTimeout):
|
|
aiextract_run.wait_for_reply('http://fake', 'sess-1', 0, timeout_s=5,
|
|
transport=transport, sleep=clock.sleep,
|
|
clock=clock.time, poll_interval=1)
|
|
|
|
|
|
def test_wait_for_reply_returns_once_session_list_says_not_running():
|
|
# THE REAL DONE-SIGNAL (measured 2026-09-03): session.history carries no
|
|
# status/state field at all -- session.list's `running` flag is what
|
|
# decides. This used to assert a 'status' key on session.history's value
|
|
# decided things; that was the old, wrong guess (the same one that
|
|
# produced the 'invalid payload for session.prompt' failure on every
|
|
# prior run) and is rewritten here to the measured contract.
|
|
clock = _FakeClock()
|
|
calls = {'n': 0}
|
|
|
|
def transport(_url, body, _timeout):
|
|
method = body['method']
|
|
if method == 'session.history':
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'events': [_assistant_event('the reply text')],
|
|
'hasMore': False, 'projections': {}}}}
|
|
if method == 'session.list':
|
|
calls['n'] += 1
|
|
running = calls['n'] == 1
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'items': [{'sessionId': 'sess-1', 'running': running}]}}}
|
|
raise AssertionError('unexpected method: ' + method)
|
|
|
|
text = aiextract_run.wait_for_reply('http://fake', 'sess-1', 0, timeout_s=30,
|
|
transport=transport, sleep=clock.sleep,
|
|
clock=clock.time, poll_interval=1)
|
|
assert text == 'the reply text'
|
|
assert calls['n'] >= 2
|
|
|
|
|
|
def test_wait_for_reply_uses_stability_when_session_list_cannot_tell():
|
|
# session_running() returns None (session not listed here at all) --
|
|
# wait_for_reply must fall back to the old stability heuristic rather
|
|
# than treating "can't tell" as "still running" or "done".
|
|
clock = _FakeClock()
|
|
|
|
def transport(_url, body, _timeout):
|
|
method = body['method']
|
|
if method == 'session.history':
|
|
return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {
|
|
'events': [_assistant_event('stable text')],
|
|
'hasMore': False, 'projections': {}}}}
|
|
if method == 'session.list':
|
|
return {'rpcId': body['rpcId'],
|
|
'result': {'ok': True, 'value': {'items': []}}}
|
|
raise AssertionError('unexpected method: ' + method)
|
|
|
|
text = aiextract_run.wait_for_reply('http://fake', 'sess-1', 0, timeout_s=30,
|
|
transport=transport, sleep=clock.sleep,
|
|
clock=clock.time, poll_interval=1)
|
|
assert text == 'stable text'
|
|
|
|
|
|
# ------------------------------------------------------------------ run()
|
|
#
|
|
# End to end, with the HTTP boundary faked -- see the module docstring.
|
|
|
|
def _assistant_event(text, turn=1, step=1):
|
|
"""One `assistant/message` history event, in the real measured shape."""
|
|
return {'event': {'type': 'assistant/message', 'data': {
|
|
'turn': turn, 'step': step,
|
|
'message': {'role': 'assistant',
|
|
'content': [{'type': 'text', 'text': text}]}}}}
|
|
|
|
|
|
def _context_event(provider, model):
|
|
"""One `request/context` history event -- MEASURED live, 2026-09-03,
|
|
against a real dsh: the event that says what a turn ACTUALLY ran on,
|
|
which `verify_session_model()` checks before `run()` ever trusts a
|
|
reply. See the module docstring's point 4.
|
|
"""
|
|
return {'event': {'type': 'request/context', 'data': {
|
|
'provider': provider, 'model': model, 'contextWindow': 1000000}}}
|
|
|
|
|
|
class _FakeDsh(object):
|
|
"""Scripted responses for session.create / .prompt / .history / .list.
|
|
|
|
`history_batches` is a list of event-lists returned in order, one per
|
|
`session.history` call (repeating the last entry once exhausted). The
|
|
session is reported `running: True` on `session.list` until the LAST
|
|
batch has been served, then `running: False` -- that is the real
|
|
done-signal (`session.history` itself carries no status field at all,
|
|
per the measured 2026-09-03 contract), so `wait_for_reply` settles once
|
|
the last batch is out and a non-empty candidate exists.
|
|
"""
|
|
def __init__(self, history_batches, session_id='sess-1'):
|
|
self.history_batches = history_batches
|
|
self.session_id = session_id
|
|
self.calls = []
|
|
self._history_calls = 0
|
|
|
|
def transport(self, url, body, _timeout):
|
|
self.calls.append((url, body))
|
|
method = body['method']
|
|
rpc_id = body['rpcId']
|
|
# The real dsh envelope, confirmed against a live session.create
|
|
# reply (owner, WPA5/sample_job run, 2026-09-01) -- see _rpc_value's
|
|
# docstring: 'result' -> 'ok' / 'value', not a flat 'result'.
|
|
if method == 'session.create':
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value':
|
|
{'sessionId': self.session_id, 'agentPreset': 'standard'}}}
|
|
if method == 'session.selectModel':
|
|
payload = body['payload']
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value':
|
|
{'selected': {'provider': payload['provider'],
|
|
'model': payload['model']}}}}
|
|
if method == 'session.prompt':
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value':
|
|
{'accepted': True}}}
|
|
if method == 'session.history':
|
|
idx = min(self._history_calls, len(self.history_batches) - 1)
|
|
self._history_calls += 1
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value':
|
|
{'events': self.history_batches[idx], 'hasMore': False,
|
|
'projections': {}}}}
|
|
if method == 'session.list':
|
|
served_last = self._history_calls >= len(self.history_batches)
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {
|
|
'items': [{'sessionId': self.session_id,
|
|
'running': not served_last}]}}}
|
|
if method == 'session.cancel':
|
|
return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {}}}
|
|
raise AssertionError('unexpected dsh method: ' + method)
|
|
|
|
|
|
def _job(tmp_path, spec, evidence=None):
|
|
spec_path = tmp_path / 'PNL1.yaml'
|
|
_write_spec(str(spec_path), spec)
|
|
evidence_path = tmp_path / 'PNL1.evidence.json'
|
|
with io.open(str(evidence_path), 'w', encoding='utf-8') as fh:
|
|
fh.write(json.dumps(evidence or {'design_number': 'PNL1'}))
|
|
return {
|
|
'spec_path': str(spec_path),
|
|
'evidence_path': str(evidence_path),
|
|
'model': 'test-vision-model',
|
|
'provider': 'test-provider',
|
|
'dsh_base_url': 'http://fake-dsh',
|
|
'timeout_s': 30,
|
|
'datum_clause': 'the LEFT rail (datum = "left_rail").',
|
|
'image_paths': [],
|
|
}
|
|
|
|
|
|
_REPLY_JSON = {'concrete': {'fci_psi': 3500},
|
|
'confidence': {'concrete.fci_psi': 'high'}}
|
|
_REPLY_TEXT = ('```json\n' + json.dumps(_REPLY_JSON) + '\n```\n\n'
|
|
'Summary: filled fci_psi from the drawing.')
|
|
|
|
|
|
def test_run_success_merges_and_stamps_provenance(tmp_path):
|
|
spec = _blank_spec()
|
|
spec['concrete']['fc_psi'] = 6000
|
|
job = _job(tmp_path, spec)
|
|
|
|
fake = _FakeDsh(history_batches=[
|
|
[], # baseline
|
|
[_context_event('test-provider', 'test-vision-model'), # settled
|
|
_assistant_event(_REPLY_TEXT)],
|
|
])
|
|
|
|
result = aiextract_run.run(job, transport=fake.transport,
|
|
sleep=lambda _s: None)
|
|
|
|
assert result['ok'] is True
|
|
# 'confidence' is metadata ABOUT a filled field, not a field in its own
|
|
# right -- it never appears in `filled` itself, only alongside it.
|
|
assert result['filled'] == ['concrete.fci_psi']
|
|
|
|
written = _read_spec(job['spec_path'])
|
|
assert written['concrete']['fc_psi'] == 6000 # untouched
|
|
assert written['concrete']['fci_psi'] == 3500 # filled
|
|
assert written['source']['extractor'] == 'dsv4f-vision'
|
|
assert 'concrete.fci_psi' in written['source']['ai_filled']
|
|
|
|
|
|
def test_run_failure_leaves_spec_field_values_untouched(tmp_path):
|
|
spec = _blank_spec()
|
|
spec['concrete']['fc_psi'] = 6000
|
|
spec['concrete']['fci_psi'] = None
|
|
job = _job(tmp_path, spec)
|
|
|
|
def failing_transport(_url, _body, _timeout):
|
|
raise aiextract_run.DshTimeout('connection refused')
|
|
|
|
result = aiextract_run.run(job, transport=failing_transport,
|
|
sleep=lambda _s: None)
|
|
|
|
assert result['ok'] is False
|
|
written = _read_spec(job['spec_path'])
|
|
assert written['concrete']['fc_psi'] == 6000
|
|
assert written['concrete']['fci_psi'] is None
|
|
assert written['source']['extractor'] == 'regex' # not restamped
|
|
assert len(written['unresolved']) == 1
|
|
assert written['unresolved'][0]['text'] == aiextract_run.FAILURE_REASON_PREFIX
|
|
|
|
|
|
def test_run_unparsable_reply_is_a_failure_not_a_silent_success(tmp_path):
|
|
spec = _blank_spec()
|
|
job = _job(tmp_path, spec)
|
|
fake = _FakeDsh(history_batches=[
|
|
[],
|
|
[_context_event('test-provider', 'test-vision-model'),
|
|
_assistant_event('I looked, but no JSON here.')],
|
|
])
|
|
|
|
result = aiextract_run.run(job, transport=fake.transport,
|
|
sleep=lambda _s: None)
|
|
|
|
assert result['ok'] is False
|
|
written = _read_spec(job['spec_path'])
|
|
assert written['source']['extractor'] == 'regex'
|
|
assert any(e['text'] == aiextract_run.FAILURE_REASON_PREFIX
|
|
for e in written['unresolved'])
|
|
|
|
|
|
def test_run_rejects_a_reply_stamped_with_the_wrong_rpc_id(tmp_path):
|
|
spec = _blank_spec()
|
|
job = _job(tmp_path, spec)
|
|
|
|
def mismatched_transport(_url, body, _timeout):
|
|
if body['method'] == 'session.create':
|
|
return {'rpcId': 'not-the-id-that-was-sent',
|
|
'result': {'sessionId': 'sess-1'}}
|
|
return {'rpcId': body['rpcId'], 'result': {'messages': []}}
|
|
|
|
result = aiextract_run.run(job, transport=mismatched_transport,
|
|
sleep=lambda _s: None)
|
|
assert result['ok'] is False
|
|
written = _read_spec(job['spec_path'])
|
|
assert written['source']['extractor'] == 'regex'
|