# -*- coding: utf-8 -*- """Tests for tools/aiextract_run.py -- the DSV4F-vision half of Design Extract. Loaded by path, like `test_knowledgelint.py` and `test_scrub.py`: the tool is host CPython (it needs a real HTTP client and, eventually, a real dsh) and lives outside `WillTools.extension` entirely. What is tested here is the part that makes an unsupervised, detached process safe to run at all: `apply_ai_fill` (never overwrite a value the mechanical reader placed; only fill what was empty, or add distinct new lines to the two collections of independent entries) and `apply_failure_note` (a run that cannot complete leaves every field exactly as it found them and says why in one line). The dsh round trip is exercised with the HTTP boundary faked -- `transport` is a plain callable substituted for `urllib.request.urlopen`, per `_rpc`'s own docstring -- never a real socket. The dsh wire format itself (payload shapes, whether `session.history` really carries a status field) is UNVERIFIED against a live dsh; see aiextract_run.py's module docstring. That gap, and the Revit-side image export in the button, are on the owner-must-verify list, not here. """ import io import json import os import sys import pytest ROOT = os.path.dirname(os.path.dirname(os.path.dirname( os.path.abspath(__file__)))) TOOLS = os.path.join(ROOT, 'tools') if TOOLS not in sys.path: sys.path.insert(0, TOOLS) aiextract_run = pytest.importorskip('aiextract_run') from vibe import specformat # --------------------------------------------------------------- fixtures def _blank_spec(design='PNL1'): """A spec shaped the way `designspec.build_spec` actually leaves one -- schema set, design number set, source.extractor stamped 'regex' -- so a test asserting the AI pass left provenance alone has a real provenance to compare against instead of the raw template's all-None source block. """ spec = json.loads(json.dumps(specformat.SPEC_TEMPLATE)) spec['design_number'] = design spec['schema'] = specformat.SCHEMA_VERSION spec['source']['extractor'] = 'regex' return spec def _write_spec(path, spec): with io.open(path, 'w', encoding='utf-8') as fh: fh.write(specformat.spec_text(spec)) def _read_spec(path): with io.open(path, 'r', encoding='utf-8') as fh: return specformat.normalize_spec(specformat.parse_spec_text(fh.read())) # ------------------------------------------------------------ apply_ai_fill def test_never_overwrites_a_scalar_the_regex_reader_already_placed(): spec = _blank_spec() spec['concrete']['fc_psi'] = 6000 ai = {'concrete': {'fc_psi': 9999, 'fci_psi': 3500}} merged, filled = aiextract_run.apply_ai_fill(spec, ai) assert merged['concrete']['fc_psi'] == 6000 # untouched assert merged['concrete']['fci_psi'] == 3500 # filled assert filled == ['concrete.fci_psi'] def test_fills_a_blank_scalar(): spec = _blank_spec() ai = {'thickness_in': 8.0} merged, filled = aiextract_run.apply_ai_fill(spec, ai) assert merged['thickness_in'] == 8.0 assert filled == ['thickness_in'] def test_does_not_fill_when_ai_proposes_nothing(): spec = _blank_spec() merged, filled = aiextract_run.apply_ai_fill(spec, {}) assert merged == spec assert filled == [] def test_structured_list_filled_only_when_currently_empty(): spec = _blank_spec() ai = {'reinforcing': {'strand_layout': { 'columns_in': [3.5, 16.5, 40.5], 'rows': [{'from_bottom_in': 1.75, 'strands': [True, True, True]}], 'total_in': 44.0, }}} merged, filled = aiextract_run.apply_ai_fill(spec, ai) layout = merged['reinforcing']['strand_layout'] assert layout['columns_in'] == [3.5, 16.5, 40.5] assert layout['total_in'] == 44.0 assert 'reinforcing.strand_layout.columns_in' in filled assert 'reinforcing.strand_layout.rows' in filled assert 'reinforcing.strand_layout.total_in' in filled def test_structured_list_never_touched_once_non_empty(): spec = _blank_spec() placed = [{'qty': 14, 'dia_in': 0.5, 'grade_ksi': 270, 'it_lb': 31600, 'role': 'main', 'note': None}] spec['reinforcing']['strand_groups'] = placed ai = {'reinforcing': {'strand_groups': [ {'qty': 99, 'dia_in': 0.6, 'grade_ksi': 300, 'it_lb': 1, 'role': 'bogus', 'note': None}]}} merged, filled = aiextract_run.apply_ai_fill(spec, ai) assert merged['reinforcing']['strand_groups'] == placed assert filled == [] def test_datum_is_never_a_content_path_at_all(): # The datum is plant knowledge, never something the model may set -- # apply_ai_fill must not even have a rule for it under any name. assert not any('datum' in p for p in aiextract_run.CONTENT_PATHS) assert not any('datum' in p for p in aiextract_run.STRUCTURED_LIST_PATHS) def test_atomic_lists_append_new_distinct_entries_and_keep_old_ones(): spec = _blank_spec() spec['unresolved'] = [{'text': 'STRIPPING DETAIL < 25\'', 'reason': 'prose'}] ai = {'unresolved': [ {'text': 'STRIPPING DETAIL < 25\'', 'reason': 'prose'}, # duplicate {'text': 'chain total disagrees with piece width by 1/4"', 'reason': 'flagged by vision pass'}, ]} merged, filled = aiextract_run.apply_ai_fill(spec, ai) assert len(merged['unresolved']) == 2 assert merged['unresolved'][0]['text'] == 'STRIPPING DETAIL < 25\'' assert merged['unresolved'][1]['text'].startswith('chain total') assert 'unresolved' in filled def test_atomic_list_merge_is_case_and_whitespace_tolerant_for_dedupe(): spec = _blank_spec() spec['notes'] = ['USE MC STANDARD REINFORCEMENT FOR CUT DOWN PANELS'] ai = {'notes': ['use mc standard reinforcement for cut down panels']} merged, filled = aiextract_run.apply_ai_fill(spec, ai) assert len(merged['notes']) == 1 assert filled == [] def test_confidence_accepted_only_for_paths_actually_just_filled(): spec = _blank_spec() spec['concrete']['fc_psi'] = 6000 ai = { 'concrete': {'fci_psi': 3500}, # This one names an ALREADY-FILLED field, so it must be dropped even # though the model is "confident" about it -- it did not just fill it. 'confidence': {'concrete.fci_psi': 'high', 'concrete.fc_psi': 'high'}, } merged, _filled = aiextract_run.apply_ai_fill(spec, ai) assert merged['confidence'] == {'concrete.fci_psi': 'high'} def test_apply_ai_fill_never_mutates_its_input(): spec = _blank_spec() spec['concrete']['fc_psi'] = 6000 before = json.loads(json.dumps(spec)) aiextract_run.apply_ai_fill(spec, {'concrete': {'fci_psi': 3500}}) assert spec == before # --------------------------------------------------------- apply_failure_note def test_failure_note_leaves_every_field_value_untouched(): spec = _blank_spec() spec['concrete']['fc_psi'] = 6000 spec['unresolved'] = [{'text': 'foo', 'reason': 'bar'}] result = aiextract_run.apply_failure_note(spec, 'DshTimeout: no reply within 180s') assert result['concrete']['fc_psi'] == 6000 assert result['source'] == spec['source'] # untouched, no stamp assert result['unresolved'][0] == {'text': 'foo', 'reason': 'bar'} assert len(result['unresolved']) == 2 assert result['unresolved'][1]['text'] == aiextract_run.FAILURE_REASON_PREFIX assert 'DshTimeout' in result['unresolved'][1]['reason'] def test_failure_note_does_not_pile_up_on_repeated_identical_failures(): spec = _blank_spec() once = aiextract_run.apply_failure_note(spec, 'same reason') twice = aiextract_run.apply_failure_note(once, 'same reason') assert len(twice['unresolved']) == 1 # ------------------------------------------------------- extract_json_payload def test_extract_json_payload_reads_a_fenced_block(): reply = ('Here is what I found.\n\n' '```json\n{"concrete": {"fci_psi": 3500}}\n```\n\n' 'Summary: filled fci_psi.') payload = aiextract_run.extract_json_payload(reply) assert payload == {'concrete': {'fci_psi': 3500}} def test_extract_json_payload_falls_back_to_first_balanced_object(): reply = 'no fence here, just {"thickness_in": 8.0} inline.' assert aiextract_run.extract_json_payload(reply) == {'thickness_in': 8.0} def test_extract_json_payload_raises_rather_than_returning_empty(): with pytest.raises(ValueError): aiextract_run.extract_json_payload('I could not find anything.') def test_extract_json_payload_rejects_a_non_object_json_value(): with pytest.raises(ValueError): aiextract_run.extract_json_payload('```json\n[1, 2, 3]\n```') # ----------------------------------------------------------------- prompt def test_build_prompt_includes_datum_and_evidence_and_template(): spec = _blank_spec() prompt = aiextract_run.build_prompt( evidence={'design_number': 'PNL1'}, regex_spec=spec, datum_clause='the LEFT rail (datum = "left_rail").', image_paths=[]) assert 'the LEFT rail' in prompt assert '"design_number": "PNL1"' in prompt assert 'strand_layout' in prompt assert 'no drawing image was available' in prompt.lower() def test_build_prompt_lists_image_paths_when_given(): spec = _blank_spec() prompt = aiextract_run.build_prompt( evidence={}, regex_spec=spec, datum_clause='d', image_paths=['C:\\job\\PNL1.summary.png']) assert 'C:\\job\\PNL1.summary.png' in prompt assert 'read_image' in prompt def test_build_prompt_strand_chain_rules_cover_concatenation_and_rows(): # rule 4a's "How to read the chain" sub-list, added after the real 10" # slab run that gave up on one 12-segment chain against a stated 19 # strands -- the drawing actually carried two rows, each with its own # chain, and the bottom chain arrived as four separate dimension # elements that needed concatenating. Each assertion below is a short, # distinctive phrase so a wording tweak elsewhere in the rule does not # break the test. spec = _blank_spec() prompt = aiextract_run.build_prompt( evidence={}, regex_spec=spec, datum_clause='d', image_paths=[]) # (a) same-y dimensions concatenate into one chain. assert 'concatenate them left to right' in prompt # (b) one chain per row; a full grid shares a single chain. assert 'One chain per row' in prompt assert 'single chain shared by every row' in prompt # (c) a chain whose total matches neither the width nor a row is ignored. assert 'NOT a strand chain' in prompt # count check against the stated quantity (and symbols, when present). assert 'MUST equal the stated quantity' in prompt # the STRAND PATTERN (matrix) unresolved-reason pointer. assert 'STRAND PATTERN (matrix)' in prompt def _leaf_paths(node, prefix=''): paths = [] for key, value in node.items(): path = '{0}.{1}'.format(prefix, key) if prefix else key if isinstance(value, dict) and value: paths.extend(_leaf_paths(value, path)) else: paths.append(path) return paths def test_reply_template_matches_the_merge_rule_tables_exactly(): # The old opus-CLI prompt drifted from the spec's real field names # (`spacings_in` / `positions_in` / `strands_per_row`, which stopped # existing when the matrix shape landed) because nothing exercised it. # This pins AI_REPLY_TEMPLATE -- what the prompt actually asks the model # to send back -- against the three tables apply_ai_fill actually reads. leaves = set(_leaf_paths(aiextract_run.AI_REPLY_TEMPLATE)) expected = (set(aiextract_run.CONTENT_PATHS) | set(aiextract_run.ATOMIC_LIST_PATHS) | {'confidence'}) assert leaves == expected # -------------------------------------------------------------- resolve_model # # `resolve_model()` now returns `(model, provider, why)` -- see the module # docstring's point 4: `session.selectModel` needs an explicit provider id # alongside the model id, and this file was rewritten to fail closed on # either rather than guess. Every test here stubs `_find_provider_for_model_id` # unless it is specifically testing provider resolution, so the model-only # assertions above stay isolated from the real ~/.dsh/settings.yaml this # machine happens to have -- the same isolation `_scan_settings_for_vision_model` # is already stubbed for below. def test_resolve_model_prefers_an_explicit_job_setting(monkeypatch): monkeypatch.delenv(aiextract_run.MODEL_ENV, raising=False) monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False) monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id', lambda *_a, **_k: ('stub-provider', None)) model, provider, why = aiextract_run.resolve_model( {'model': 'named-explicitly'}) assert model == 'named-explicitly' assert provider == 'stub-provider' assert why is None def test_resolve_model_falls_back_to_the_env_var(monkeypatch): monkeypatch.setenv(aiextract_run.MODEL_ENV, 'from-env') monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False) monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id', lambda *_a, **_k: ('stub-provider', None)) model, provider, why = aiextract_run.resolve_model({}) assert model == 'from-env' assert provider == 'stub-provider' assert why is None def test_resolve_model_uses_settings_scan_result(monkeypatch): monkeypatch.delenv(aiextract_run.MODEL_ENV, raising=False) monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False) monkeypatch.setattr(aiextract_run, '_scan_settings_for_vision_model', lambda: ('commandcode-vision', 'commandcode', None)) model, provider, why = aiextract_run.resolve_model({}) assert model == 'commandcode-vision' # The scan already knew which provider hosts it -- free, no separate # `_find_provider_for_model_id` lookup needed. assert provider == 'commandcode' assert why is None def test_resolve_model_falls_back_to_unconfirmed_constant_and_says_so(monkeypatch): monkeypatch.delenv(aiextract_run.MODEL_ENV, raising=False) monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False) monkeypatch.setattr(aiextract_run, '_scan_settings_for_vision_model', lambda: (None, None, 'no settings file')) monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id', lambda *_a, **_k: (None, 'no settings file')) model, provider, why = aiextract_run.resolve_model({}) assert model == aiextract_run.FALLBACK_VISION_MODEL assert provider is None assert why is not None and 'unconfirmed' in why # --------------------------------------------------- resolve_model: provider def test_resolve_model_prefers_an_explicit_job_provider(monkeypatch): monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False) def _boom(*_a, **_k): raise AssertionError('should not need to look the provider up') monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id', _boom) model, provider, why = aiextract_run.resolve_model( {'model': 'named-explicitly', 'provider': 'named-provider'}) assert model == 'named-explicitly' assert provider == 'named-provider' assert why is None def test_resolve_model_falls_back_to_the_provider_env_var(monkeypatch): monkeypatch.setenv(aiextract_run.PROVIDER_ENV, 'env-provider') def _boom(*_a, **_k): raise AssertionError('should not need to look the provider up') monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id', _boom) model, provider, why = aiextract_run.resolve_model( {'model': 'named-explicitly'}) assert provider == 'env-provider' assert why is None def test_resolve_model_refuses_to_guess_a_provider_it_cannot_find(monkeypatch): # THE FAIL-CLOSED CASE: a model id came from somewhere, but nothing # (job, env, settings.yaml) can say which provider serves it -- # `session.selectModel` requires one, so this must never invent it. monkeypatch.delenv(aiextract_run.PROVIDER_ENV, raising=False) monkeypatch.setattr(aiextract_run, '_find_provider_for_model_id', lambda *_a, **_k: (None, 'no entry for this model')) model, provider, why = aiextract_run.resolve_model( {'model': 'named-explicitly'}) assert model == 'named-explicitly' assert provider is None assert why is not None and 'provider' in why.lower() # The REAL shape, mimicked. Coordinator, 2026-09-01, quoting the actual file # on the machine this ships to (`~/.dsh/settings.yaml`, line 407ff): an `id:` # list item and its `input:` list are SEVERAL LINES APART, separated by other # keys, inside a deeply nested `providers: [...] -> models: [...]` structure # -- not the flat `mapping-key: {input: [...]}` shape the old line-scan # heuristic (and the tests that used to live here) assumed. This fixture is # that shape, and it is what the parser is actually judged against now. _REAL_SETTINGS_FIXTURE = ( 'version: 1\n' 'providers:\n' ' - name: commandcode\n' ' base_url: https://example.invalid\n' ' models:\n' ' - id: deepseek/deepseek-v4-flash\n' ' display_name: DeepSeek V4 Flash\n' ' context_window: 128000\n' ' pricing:\n' ' input_per_mtok: 0.1\n' ' output_per_mtok: 0.3\n' ' input:\n' ' - text\n' ' - id: deepseek/deepseek-v4-flash-vision-exp\n' ' display_name: DeepSeek V4 Flash Vision (Experimental)\n' ' context_window: 128000\n' ' pricing:\n' ' input_per_mtok: 0.15\n' ' output_per_mtok: 0.4\n' ' off_peak_discount: 0.5\n' ' capabilities:\n' ' - vision\n' ' - tool_use\n' ' input:\n' ' - text\n' ' - image\n' ' output:\n' ' - text\n' ) def test_scan_settings_finds_the_real_deeply_nested_entry(tmp_path): settings = tmp_path / 'settings.yaml' settings.write_text(_REAL_SETTINGS_FIXTURE, encoding='utf-8') model, provider, why = aiextract_run._scan_settings_for_vision_model( str(settings)) assert model == 'deepseek/deepseek-v4-flash-vision-exp' assert provider == 'commandcode' # from the provider's `name:` assert why is None def test_resolve_model_finds_the_real_entry_end_to_end(tmp_path, monkeypatch): settings = tmp_path / 'settings.yaml' settings.write_text(_REAL_SETTINGS_FIXTURE, encoding='utf-8') monkeypatch.delenv(aiextract_run.MODEL_ENV, raising=False) original_scan = aiextract_run._scan_settings_for_vision_model monkeypatch.setattr(aiextract_run, '_scan_settings_for_vision_model', lambda: original_scan(str(settings))) model, provider, why = aiextract_run.resolve_model({}) assert model == 'deepseek/deepseek-v4-flash-vision-exp' assert provider == 'commandcode' assert why is None # The OTHER real shape. Confirmed live, 2026-09-03, reading this machine's # actual `~/.dsh/settings.yaml`: `providers` there is not a list at all -- # it is a MAPPING keyed by provider id (`llm-pi-ai: providers: {commandcode: # {models: [...]}, opencode: {...}}`), and `commandcode` (the dict key) is # exactly the string `session.selectModel` needs as `provider`. Both shapes # have to resolve a provider id; `_iter_model_entries` is what tolerates # either (see its docstring). _REAL_DICT_KEYED_PROVIDERS_FIXTURE = ( 'llm-pi-ai:\n' ' providers:\n' ' commandcode:\n' ' displayName: Command Code API\n' ' baseURL: https://api.commandcode.ai/provider/v1\n' ' models:\n' ' - id: deepseek/deepseek-v4-flash\n' ' name: DeepSeek V4 Flash (latest)\n' ' - id: deepseek/deepseek-v4-flash-vision-exp\n' ' name: DeepSeek V4 Flash Vision (exp)\n' ' input:\n' ' - text\n' ' - image\n' ) def test_scan_settings_finds_the_provider_id_when_providers_is_a_dict(tmp_path): settings = tmp_path / 'settings.yaml' settings.write_text(_REAL_DICT_KEYED_PROVIDERS_FIXTURE, encoding='utf-8') model, provider, why = aiextract_run._scan_settings_for_vision_model( str(settings)) assert model == 'deepseek/deepseek-v4-flash-vision-exp' assert provider == 'commandcode' # the dict KEY, not a `name:` assert why is None def test_scan_settings_picks_the_vision_named_id_when_several_qualify(tmp_path): # Two entries both declare image input; only one's id says "vision" -- # that one wins without asking, per the owner's own naming convention. settings = tmp_path / 'settings.yaml' settings.write_text( 'providers:\n' ' - models:\n' ' - id: some/multimodal-model\n' ' input: [text, image]\n' ' - id: some/vision-model\n' ' input: [text, image]\n', encoding='utf-8') model, _provider, why = aiextract_run._scan_settings_for_vision_model( str(settings)) assert model == 'some/vision-model' assert why is None def test_scan_settings_refuses_to_guess_between_two_candidates(tmp_path): settings = tmp_path / 'settings.yaml' settings.write_text( 'providers:\n' ' - models:\n' ' - id: vendor/vision-a\n' ' input: [text, image]\n' ' - id: vendor/vision-b\n' ' input: [text, image]\n', encoding='utf-8') model, provider, why = aiextract_run._scan_settings_for_vision_model( str(settings)) assert model is None assert provider is None assert 'vendor/vision-a' in why and 'vendor/vision-b' in why def test_scan_settings_missing_file_says_so(tmp_path): model, provider, why = aiextract_run._scan_settings_for_vision_model( str(tmp_path / 'does-not-exist.yaml')) assert model is None assert provider is None assert 'no settings file' in why def test_scan_settings_ignores_entries_with_id_but_no_image_input(tmp_path): settings = tmp_path / 'settings.yaml' settings.write_text(_REAL_SETTINGS_FIXTURE, encoding='utf-8') model, _provider, _why = aiextract_run._scan_settings_for_vision_model( str(settings)) assert model != 'deepseek/deepseek-v4-flash' # the text-only sibling def test_scan_settings_malformed_yaml_says_so_not_raises(tmp_path): settings = tmp_path / 'settings.yaml' settings.write_text('providers: [this is not: valid: yaml: at all\n', encoding='utf-8') model, provider, why = aiextract_run._scan_settings_for_vision_model( str(settings)) assert model is None assert provider is None assert 'could not parse' in why # --------------------------------------------------- _find_provider_for_model_id def test_find_provider_for_model_id_dict_keyed_shape(tmp_path): settings = tmp_path / 'settings.yaml' settings.write_text(_REAL_DICT_KEYED_PROVIDERS_FIXTURE, encoding='utf-8') provider, why = aiextract_run._find_provider_for_model_id( 'deepseek/deepseek-v4-flash', str(settings)) assert provider == 'commandcode' assert why is None def test_find_provider_for_model_id_ignores_whether_it_declares_image_input( tmp_path): # Unlike `_scan_settings_for_vision_model`, this does not care about # vision -- it is asked about a model the CALLER already chose (an # explicit job/env override), and just needs to know who serves it. settings = tmp_path / 'settings.yaml' settings.write_text(_REAL_DICT_KEYED_PROVIDERS_FIXTURE, encoding='utf-8') provider, why = aiextract_run._find_provider_for_model_id( 'deepseek/deepseek-v4-flash-vision-exp', str(settings)) assert provider == 'commandcode' assert why is None def test_find_provider_for_model_id_not_found_says_so(tmp_path): settings = tmp_path / 'settings.yaml' settings.write_text(_REAL_DICT_KEYED_PROVIDERS_FIXTURE, encoding='utf-8') provider, why = aiextract_run._find_provider_for_model_id( 'no/such-model', str(settings)) assert provider is None assert 'no/such-model' in why def test_find_provider_for_model_id_missing_file_says_so(tmp_path): provider, why = aiextract_run._find_provider_for_model_id( 'anything', str(tmp_path / 'does-not-exist.yaml')) assert provider is None assert 'no settings file' in why def test_find_provider_for_model_id_refuses_to_guess_between_two_providers( tmp_path): settings = tmp_path / 'settings.yaml' settings.write_text( 'providers:\n' ' vendor-a:\n' ' models:\n' ' - id: shared/model-id\n' ' vendor-b:\n' ' models:\n' ' - id: shared/model-id\n', encoding='utf-8') provider, why = aiextract_run._find_provider_for_model_id( 'shared/model-id', str(settings)) assert provider is None assert 'vendor-a' in why and 'vendor-b' in why # --------------------------------------------------------------------- _rpc_value # # The bug: a real WPA5/sample_job run got `DshError: session.create did not # return a session id` on a call that had, in fact, fully succeeded -- # `create_session` was reading `result.sessionId` when the real reply nests # the id two deep, at `result.value.sessionId`, and the session it could not # see was left orphaned, live, on the dsh server. This section pins the # EXACT captured response (the regression) and every malformed variant the # task called out, each naming the layer that failed rather than crashing. _REAL_SESSION_CREATE_RESPONSE = { 'type': 'server-response', 'rpcId': '3d7ad657a9b4a83a5234cfd5ce185c7', 'result': {'ok': True, 'value': { 'sessionId': 'session-5969e01e-abe1-4182-a7a3-6c2f594c5419', 'agentPreset': 'standard'}}, } def test_rpc_value_parses_the_exact_captured_session_create_response(): # THE REGRESSION. This is the literal response from the failed WPA5 run # (owner report, 2026-09-01), byte-for-byte down to the rpcId. value = aiextract_run._rpc_value(_REAL_SESSION_CREATE_RESPONSE, 'session.create') assert value == { 'sessionId': 'session-5969e01e-abe1-4182-a7a3-6c2f594c5419', 'agentPreset': 'standard'} def test_create_session_parses_the_exact_captured_response(): # Same 'result' as the real captured reply, echoing whatever rpcId this # call actually sends (create_session itself has no control over the # rpcId `_rpc` generates, so the transport must echo it, same as every # other fake transport in this file -- only the 'result' body below is # the real, captured one). def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': _REAL_SESSION_CREATE_RESPONSE['result']} session_id = aiextract_run.create_session('http://fake', 30, transport) assert session_id == 'session-5969e01e-abe1-4182-a7a3-6c2f594c5419' def test_rpc_value_refuses_when_result_ok_is_false(): res = {'rpcId': 'x', 'result': {'ok': False, 'value': {'sessionId': 'a'}}} with pytest.raises(aiextract_run.DshError) as exc: aiextract_run._rpc_value(res, 'session.create') assert 'result.ok' in str(exc.value) def test_rpc_value_refuses_when_value_is_missing(): res = {'rpcId': 'x', 'result': {'ok': True}} with pytest.raises(aiextract_run.DshError) as exc: aiextract_run._rpc_value(res, 'session.create') assert 'result.value' in str(exc.value) def test_rpc_value_refuses_when_value_is_not_an_object(): res = {'rpcId': 'x', 'result': {'ok': True, 'value': 'not-an-object'}} with pytest.raises(aiextract_run.DshError) as exc: aiextract_run._rpc_value(res, 'session.create') assert 'result.value' in str(exc.value) def test_rpc_value_refuses_when_result_is_missing_entirely(): res = {'rpcId': 'x'} with pytest.raises(aiextract_run.DshError) as exc: aiextract_run._rpc_value(res, 'session.create') assert '"result"' in str(exc.value) def test_create_session_refuses_clearly_on_the_old_flat_wrong_shape(): # The shape the FIRST version of this file (wrongly) expected -- # result.sessionId, one level, no 'ok'/'value' at all. Must refuse # clearly (naming which check failed), never crash with a raw KeyError # or silently return something wrong. def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'sessionId': 'sess-1'}} with pytest.raises(aiextract_run.DshError) as exc: aiextract_run.create_session('http://fake', 30, transport) assert 'result.ok' in str(exc.value) def test_create_session_names_sessionid_when_value_has_no_usable_one(): def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {'agentPreset': 'standard'}}} with pytest.raises(aiextract_run.DshError) as exc: aiextract_run.create_session('http://fake', 30, transport) assert 'sessionId' in str(exc.value) def test_create_session_refuses_an_empty_string_sessionid(): def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {'sessionId': ' '}}} with pytest.raises(aiextract_run.DshError) as exc: aiextract_run.create_session('http://fake', 30, transport) assert 'sessionId' in str(exc.value) # ----------------------------------------------------------- cancel_session def test_cancel_session_returns_true_when_the_rpc_succeeds(): calls = [] def transport(_url, body, _timeout): calls.append(body) return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {}}} assert aiextract_run.cancel_session('http://fake', 'sess-1', 30, transport) is True assert calls[0]['method'] == 'session.cancel' assert calls[0]['payload'] == {'sessionId': 'sess-1'} def test_cancel_session_swallows_errors_and_returns_false(): def transport(_url, _body, _timeout): raise aiextract_run.DshTimeout('no reply') assert aiextract_run.cancel_session('http://fake', 'sess-1', 30, transport) is False def test_run_cancels_the_session_when_something_fails_after_it_was_created( tmp_path): # The residual orphan-session risk: create_session succeeds and returns # a real, trusted id, but something LATER in the same pass fails (here, # the reply never settles). `run()` must attempt a best-effort # session.cancel with that id before writing the failure note. spec = _blank_spec() job = _job(tmp_path, spec) cancel_calls = [] def transport(_url, body, _timeout): method = body['method'] rpc_id = body['rpcId'] if method == 'session.create': return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {'sessionId': 'sess-orphan-check'}}} if method == 'session.selectModel': payload = body['payload'] return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {'selected': {'provider': payload['provider'], 'model': payload['model']}}}} if method == 'session.history': return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {'messages': []}}} if method == 'session.prompt': return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {}}} if method == 'session.cancel': cancel_calls.append(body['payload']) return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {}}} raise AssertionError('unexpected method: ' + method) # A real fake clock, not real wall time: `job.get('timeout_s') or # DEFAULT_TIMEOUT_S` means 0 would silently fall back to the 180s # default (falsy-0 is the same trap as everywhere else `or` guards a # possibly-zero setting), so this uses a small positive timeout plus a # stepped fake clock/sleep -- the same pattern `_FakeClock` exists for -- # rather than actually waiting. clock = _FakeClock() job['timeout_s'] = 5 result = aiextract_run.run(job, transport=transport, sleep=clock.sleep, clock=clock.time) assert result['ok'] is False assert cancel_calls == [{'sessionId': 'sess-orphan-check'}] def test_run_never_half_merges_on_the_exact_captured_session_create_shape( tmp_path): # THE NEVER-HALF-MERGE GUARANTEE, confirmed on THIS specific failure # mode: a session.create reply shaped exactly like the real WPA5 capture # (see _REAL_SESSION_CREATE_RESPONSE above) but with a corrupted 'ok' # flag, so it still fails the same way the original bug would have if # the fix regressed. The regex-written spec's field values, and # source.extractor, must come through byte-identical -- apply_failure_note's # contract from yesterday, re-confirmed against today's failure mode. spec = _blank_spec() spec['concrete']['fc_psi'] = 6000 spec['thickness_in'] = 8.0 job = _job(tmp_path, spec) def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': False, 'value': { 'sessionId': 'session-5969e01e-abe1-4182-a7a3-6c2f594c5419', 'agentPreset': 'standard'}}} result = aiextract_run.run(job, transport=transport, sleep=lambda _s: None) assert result['ok'] is False written = _read_spec(job['spec_path']) assert written['concrete']['fc_psi'] == 6000 assert written['thickness_in'] == 8.0 assert written['source']['extractor'] == 'regex' # never restamped assert 'ai_filled' not in written['source'] # never even added assert len(written['unresolved']) == 1 assert written['unresolved'][0]['text'] == aiextract_run.FAILURE_REASON_PREFIX assert 'result.ok' in written['unresolved'][0]['reason'] def test_run_does_not_attempt_a_cancel_when_session_create_itself_failed( tmp_path): # No session id was ever obtained, so there is nothing safe to cancel -- # attempting one here would mean guessing an id, which cancel_session's # own docstring says this script must not do. spec = _blank_spec() job = _job(tmp_path, spec) calls = [] def transport(_url, body, _timeout): calls.append(body['method']) raise aiextract_run.DshTimeout('dsh unreachable') result = aiextract_run.run(job, transport=transport, sleep=lambda _s: None) assert result['ok'] is False assert 'session.cancel' not in calls # --------------------------------------------------------------- send_prompt def test_send_prompt_sends_the_real_measured_payload_shape(): # MEASURED live against dsh, 2026-09-03: this is the exact shape that # was missing before -- the old code sent {'sessionId', 'text'} and dsh # answered 'invalid payload for session.prompt' on every run. calls = [] def transport(_url, body, _timeout): calls.append(body) return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {}}} aiextract_run.send_prompt('http://fake', 'sess-1', 'hello model', 30, transport) assert calls[0]['method'] == 'session.prompt' assert calls[0]['payload'] == { 'sessionId': 'sess-1', 'mode': 'queue', 'content': [{'type': 'text', 'text': 'hello model'}], } # --------------------------------------------------------------- select_model # # THE ACTUAL FIX: `session.create`'s real payload schema # (`sessionCreateRequestSchema`, dsh-host-apiproxy) has no `model` field at # all -- `create_session` no longer sends one (see its own docstring and the # `test_create_session_sends_no_model_field_at_all` test below). # `session.selectModel` is the separate RPC that actually pins a session's # model, and `provider` is a required field of its payload. def test_create_session_sends_no_model_field_at_all(): calls = [] def transport(_url, body, _timeout): calls.append(body) return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'sessionId': 'sess-1', 'agentPreset': 'standard'}}} aiextract_run.create_session('http://fake', 30, transport) assert calls[0]['method'] == 'session.create' assert 'model' not in calls[0]['payload'] def test_select_model_sends_the_measured_payload_shape(): # MEASURED live against dsh, 2026-09-03: session.create -> session. # selectModel({provider: "commandcode", model: # "deepseek/deepseek-v4-flash-vision-exp"}) -> session.models # immediately echoed that exact pair back as `current`. calls = [] def transport(_url, body, _timeout): calls.append(body) return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'selected': {'provider': 'commandcode', 'model': 'deepseek/deepseek-v4-flash-vision-exp'}}}} value = aiextract_run.select_model( 'http://fake', 'sess-1', 'commandcode', 'deepseek/deepseek-v4-flash-vision-exp', 30, transport) assert calls[0]['method'] == 'session.selectModel' assert calls[0]['payload'] == { 'sessionId': 'sess-1', 'provider': 'commandcode', 'model': 'deepseek/deepseek-v4-flash-vision-exp', } assert value['selected']['provider'] == 'commandcode' assert value['selected']['model'] == \ 'deepseek/deepseek-v4-flash-vision-exp' # ---------------------------------------------------------- verify_session_model # # THE CHECK THIS TASK WAS OPENED TO ADD: neither `session.create`'s response # nor `session.selectModel`'s own response is proof a turn actually ran on # the requested model -- both only echo what was asked for. The # `request/context` history event is what says what a turn ACTUALLY used. def test_verify_session_model_passes_when_context_matches(): def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'events': [_context_event( 'commandcode', 'deepseek/deepseek-v4-flash-vision-exp')], 'hasMore': False}}} provider, model = aiextract_run.verify_session_model( 'http://fake', 'sess-1', 'commandcode', 'deepseek/deepseek-v4-flash-vision-exp', 30, transport) assert provider == 'commandcode' assert model == 'deepseek/deepseek-v4-flash-vision-exp' def test_verify_session_model_raises_naming_both_on_mismatch(): # THE BUG THIS CHECK EXISTS FOR: a session can accept selectModel and # still actually run on the account's default model -- this is the one # place that would have caught the original failure (a vision request # silently served by deepseek-v4-flash, no image input). def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'events': [_context_event( 'commandcode', 'deepseek/deepseek-v4-flash')], 'hasMore': False}}} with pytest.raises(aiextract_run.DshError) as exc: aiextract_run.verify_session_model( 'http://fake', 'sess-1', 'commandcode', 'deepseek/deepseek-v4-flash-vision-exp', 30, transport) message = str(exc.value) assert 'deepseek/deepseek-v4-flash-vision-exp' in message # requested assert 'deepseek/deepseek-v4-flash' in message # actual assert 'commandcode' in message def test_verify_session_model_raises_when_no_context_event_seen_yet(): # "could not tell" must never pass as "must be fine" -- same discipline # as everywhere else in this codebase (CLAUDE.md's `knowledge.same_face()` # example). def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'events': [], 'hasMore': False}}} with pytest.raises(aiextract_run.DshError) as exc: aiextract_run.verify_session_model( 'http://fake', 'sess-1', 'commandcode', 'some-model', 30, transport) assert 'request/context' in str(exc.value) def test_verify_session_model_uses_the_last_context_event_when_several(): # A turn can carry more than one request/context event (compaction, a # retry); the LAST one is what the turn actually settled on. def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'events': [ _context_event('commandcode', 'deepseek/deepseek-v4-flash'), _context_event('commandcode', 'deepseek/deepseek-v4-flash-vision-exp'), ], 'hasMore': False}}} provider, model = aiextract_run.verify_session_model( 'http://fake', 'sess-1', 'commandcode', 'deepseek/deepseek-v4-flash-vision-exp', 30, transport) assert provider == 'commandcode' assert model == 'deepseek/deepseek-v4-flash-vision-exp' def test_extract_context_events_ignores_events_of_other_types(): value = {'events': [ {'event': {'type': 'turn/start', 'data': {}}}, {'event': {'type': 'request/context', 'data': { 'provider': 'commandcode', 'model': 'm-1'}}}, {'event': {'type': 'request/header', 'data': { 'header': {'config': {'provider': 'commandcode', 'model': 'm-1'}}}}}, ]} events = aiextract_run._extract_context_events(value) assert events == [{'provider': 'commandcode', 'model': 'm-1'}] # ----------------------------------------------------------- _history_messages def test_history_messages_projects_the_real_events_shape(): # user/message keeps its text; assistant/message keeps only its "text" # blocks and drops "reasoning" ones; assistant/chunk (a streaming delta) # is ignored entirely, same as every other event type not named here. value = { 'events': [ {'event': {'type': 'turn/start', 'data': {}}}, {'event': {'type': 'user/message', 'data': { 'content': [{'type': 'text', 'text': 'please read this'}]}}}, {'event': {'type': 'assistant/chunk', 'data': { 'chunk': {'type': 'text-delta', 'text': 'partial...'}}}}, {'event': {'type': 'assistant/message', 'data': { 'turn': 1, 'step': 1, 'message': { 'role': 'assistant', 'content': [ {'type': 'reasoning', 'text': 'thinking it through'}, {'type': 'text', 'text': 'the settled answer'}, ]}}}}, ], 'hasMore': False, 'projections': {}, } messages = aiextract_run._history_messages(value) assert messages == [ {'role': 'user', 'text': 'please read this'}, {'role': 'assistant', 'text': 'the settled answer'}, ] def test_history_messages_still_tolerates_the_old_guessed_shape(): value = {'messages': [{'role': 'assistant', 'content': 'old-shape text'}]} messages = aiextract_run._history_messages(value) assert messages == [{'role': 'assistant', 'text': 'old-shape text'}] # ------------------------------------------------------------- session_running def test_session_running_true_when_the_session_is_listed_and_running(): def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'items': [{'sessionId': 'sess-1', 'running': True}]}}} assert aiextract_run.session_running( 'http://fake', 'sess-1', 30, transport) is True def test_session_running_false_once_the_turn_has_finished(): def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'items': [{'sessionId': 'sess-1', 'running': False}]}}} assert aiextract_run.session_running( 'http://fake', 'sess-1', 30, transport) is False def test_session_running_none_when_the_session_is_not_listed(): def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'items': [{'sessionId': 'some-other-session', 'running': True}]}}} assert aiextract_run.session_running( 'http://fake', 'sess-1', 30, transport) is None # --------------------------------------------------------------- wait_for_reply class _FakeClock(object): def __init__(self, start=0.0, step=2.0): self.now = start self.step = step def time(self): return self.now def sleep(self, _interval): self.now += self.step def test_wait_for_reply_times_out_when_nothing_ever_arrives(): clock = _FakeClock() def transport(_url, body, _timeout): return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {'messages': []}}} with pytest.raises(aiextract_run.DshTimeout): aiextract_run.wait_for_reply('http://fake', 'sess-1', 0, timeout_s=5, transport=transport, sleep=clock.sleep, clock=clock.time, poll_interval=1) def test_wait_for_reply_returns_once_session_list_says_not_running(): # THE REAL DONE-SIGNAL (measured 2026-09-03): session.history carries no # status/state field at all -- session.list's `running` flag is what # decides. This used to assert a 'status' key on session.history's value # decided things; that was the old, wrong guess (the same one that # produced the 'invalid payload for session.prompt' failure on every # prior run) and is rewritten here to the measured contract. clock = _FakeClock() calls = {'n': 0} def transport(_url, body, _timeout): method = body['method'] if method == 'session.history': return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'events': [_assistant_event('the reply text')], 'hasMore': False, 'projections': {}}}} if method == 'session.list': calls['n'] += 1 running = calls['n'] == 1 return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'items': [{'sessionId': 'sess-1', 'running': running}]}}} raise AssertionError('unexpected method: ' + method) text = aiextract_run.wait_for_reply('http://fake', 'sess-1', 0, timeout_s=30, transport=transport, sleep=clock.sleep, clock=clock.time, poll_interval=1) assert text == 'the reply text' assert calls['n'] >= 2 def test_wait_for_reply_uses_stability_when_session_list_cannot_tell(): # session_running() returns None (session not listed here at all) -- # wait_for_reply must fall back to the old stability heuristic rather # than treating "can't tell" as "still running" or "done". clock = _FakeClock() def transport(_url, body, _timeout): method = body['method'] if method == 'session.history': return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': { 'events': [_assistant_event('stable text')], 'hasMore': False, 'projections': {}}}} if method == 'session.list': return {'rpcId': body['rpcId'], 'result': {'ok': True, 'value': {'items': []}}} raise AssertionError('unexpected method: ' + method) text = aiextract_run.wait_for_reply('http://fake', 'sess-1', 0, timeout_s=30, transport=transport, sleep=clock.sleep, clock=clock.time, poll_interval=1) assert text == 'stable text' # ------------------------------------------------------------------ run() # # End to end, with the HTTP boundary faked -- see the module docstring. def _assistant_event(text, turn=1, step=1): """One `assistant/message` history event, in the real measured shape.""" return {'event': {'type': 'assistant/message', 'data': { 'turn': turn, 'step': step, 'message': {'role': 'assistant', 'content': [{'type': 'text', 'text': text}]}}}} def _context_event(provider, model): """One `request/context` history event -- MEASURED live, 2026-09-03, against a real dsh: the event that says what a turn ACTUALLY ran on, which `verify_session_model()` checks before `run()` ever trusts a reply. See the module docstring's point 4. """ return {'event': {'type': 'request/context', 'data': { 'provider': provider, 'model': model, 'contextWindow': 1000000}}} class _FakeDsh(object): """Scripted responses for session.create / .prompt / .history / .list. `history_batches` is a list of event-lists returned in order, one per `session.history` call (repeating the last entry once exhausted). The session is reported `running: True` on `session.list` until the LAST batch has been served, then `running: False` -- that is the real done-signal (`session.history` itself carries no status field at all, per the measured 2026-09-03 contract), so `wait_for_reply` settles once the last batch is out and a non-empty candidate exists. """ def __init__(self, history_batches, session_id='sess-1'): self.history_batches = history_batches self.session_id = session_id self.calls = [] self._history_calls = 0 def transport(self, url, body, _timeout): self.calls.append((url, body)) method = body['method'] rpc_id = body['rpcId'] # The real dsh envelope, confirmed against a live session.create # reply (owner, WPA5/sample_job run, 2026-09-01) -- see _rpc_value's # docstring: 'result' -> 'ok' / 'value', not a flat 'result'. if method == 'session.create': return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {'sessionId': self.session_id, 'agentPreset': 'standard'}}} if method == 'session.selectModel': payload = body['payload'] return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {'selected': {'provider': payload['provider'], 'model': payload['model']}}}} if method == 'session.prompt': return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {'accepted': True}}} if method == 'session.history': idx = min(self._history_calls, len(self.history_batches) - 1) self._history_calls += 1 return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {'events': self.history_batches[idx], 'hasMore': False, 'projections': {}}}} if method == 'session.list': served_last = self._history_calls >= len(self.history_batches) return {'rpcId': rpc_id, 'result': {'ok': True, 'value': { 'items': [{'sessionId': self.session_id, 'running': not served_last}]}}} if method == 'session.cancel': return {'rpcId': rpc_id, 'result': {'ok': True, 'value': {}}} raise AssertionError('unexpected dsh method: ' + method) def _job(tmp_path, spec, evidence=None): spec_path = tmp_path / 'PNL1.yaml' _write_spec(str(spec_path), spec) evidence_path = tmp_path / 'PNL1.evidence.json' with io.open(str(evidence_path), 'w', encoding='utf-8') as fh: fh.write(json.dumps(evidence or {'design_number': 'PNL1'})) return { 'spec_path': str(spec_path), 'evidence_path': str(evidence_path), 'model': 'test-vision-model', 'provider': 'test-provider', 'dsh_base_url': 'http://fake-dsh', 'timeout_s': 30, 'datum_clause': 'the LEFT rail (datum = "left_rail").', 'image_paths': [], } _REPLY_JSON = {'concrete': {'fci_psi': 3500}, 'confidence': {'concrete.fci_psi': 'high'}} _REPLY_TEXT = ('```json\n' + json.dumps(_REPLY_JSON) + '\n```\n\n' 'Summary: filled fci_psi from the drawing.') def test_run_success_merges_and_stamps_provenance(tmp_path): spec = _blank_spec() spec['concrete']['fc_psi'] = 6000 job = _job(tmp_path, spec) fake = _FakeDsh(history_batches=[ [], # baseline [_context_event('test-provider', 'test-vision-model'), # settled _assistant_event(_REPLY_TEXT)], ]) result = aiextract_run.run(job, transport=fake.transport, sleep=lambda _s: None) assert result['ok'] is True # 'confidence' is metadata ABOUT a filled field, not a field in its own # right -- it never appears in `filled` itself, only alongside it. assert result['filled'] == ['concrete.fci_psi'] written = _read_spec(job['spec_path']) assert written['concrete']['fc_psi'] == 6000 # untouched assert written['concrete']['fci_psi'] == 3500 # filled assert written['source']['extractor'] == 'dsv4f-vision' assert 'concrete.fci_psi' in written['source']['ai_filled'] def test_run_failure_leaves_spec_field_values_untouched(tmp_path): spec = _blank_spec() spec['concrete']['fc_psi'] = 6000 spec['concrete']['fci_psi'] = None job = _job(tmp_path, spec) def failing_transport(_url, _body, _timeout): raise aiextract_run.DshTimeout('connection refused') result = aiextract_run.run(job, transport=failing_transport, sleep=lambda _s: None) assert result['ok'] is False written = _read_spec(job['spec_path']) assert written['concrete']['fc_psi'] == 6000 assert written['concrete']['fci_psi'] is None assert written['source']['extractor'] == 'regex' # not restamped assert len(written['unresolved']) == 1 assert written['unresolved'][0]['text'] == aiextract_run.FAILURE_REASON_PREFIX def test_run_unparsable_reply_is_a_failure_not_a_silent_success(tmp_path): spec = _blank_spec() job = _job(tmp_path, spec) fake = _FakeDsh(history_batches=[ [], [_context_event('test-provider', 'test-vision-model'), _assistant_event('I looked, but no JSON here.')], ]) result = aiextract_run.run(job, transport=fake.transport, sleep=lambda _s: None) assert result['ok'] is False written = _read_spec(job['spec_path']) assert written['source']['extractor'] == 'regex' assert any(e['text'] == aiextract_run.FAILURE_REASON_PREFIX for e in written['unresolved']) def test_run_rejects_a_reply_stamped_with_the_wrong_rpc_id(tmp_path): spec = _blank_spec() job = _job(tmp_path, spec) def mismatched_transport(_url, body, _timeout): if body['method'] == 'session.create': return {'rpcId': 'not-the-id-that-was-sent', 'result': {'sessionId': 'sess-1'}} return {'rpcId': body['rpcId'], 'result': {'messages': []}} result = aiextract_run.run(job, transport=mismatched_transport, sleep=lambda _s: None) assert result['ok'] is False written = _read_spec(job['spec_path']) assert written['source']['extractor'] == 'regex'