will
25592fa6c2
ci / test (push) Successful in 6m14s
History-free snapshot of the private development repository at 77a82eb. Code, docs and the public catalogue only: no plant or job knowledge layers, no client, job or producer names (tools/scrub.py clean against the widened term list; a second word-level grep clean). The intro deck is left out because its slide images cannot be checked for job drawings.
274 lines
10 KiB
Python
274 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Ask a small vision model one question about one rendered ticket view.
|
|
|
|
python tools/vision_read.py <request.json> <response.json>
|
|
|
|
This is the CPython half of the Ticket QC vision escalation. It exists as a
|
|
separate process for the same two reasons `vibe.aiextract` does:
|
|
|
|
* the extension runs on IronPython 2.7 inside Revit's UI thread, and there
|
|
is no HTTP client there worth trusting with a model call;
|
|
* nothing here touches the Revit API. It reads PNGs and JSON and writes
|
|
JSON, so it has no business being inside Revit.
|
|
|
|
The two halves talk through files, never imports. `vibe.ticketvision` builds
|
|
the request, this writes the response, and neither imports the other.
|
|
|
|
WHAT THIS MAY AND MAY NOT DO
|
|
----------------------------
|
|
It may return a verdict from a model that looked at a picture. It may not
|
|
return anything that renders like a computed result. Every row it writes
|
|
carries `source: "model-read"`, and a row it could not obtain is
|
|
`status: "NOT_RUN"` with a reason -- never a pass, never silence. Reaching
|
|
the model is not the same as the model being right, and neither is the same
|
|
as this program having run at all; the three are kept separate on purpose.
|
|
|
|
CREDENTIALS
|
|
-----------
|
|
Read from the environment by the Anthropic SDK: `ANTHROPIC_API_KEY`, or an
|
|
`ant auth login` profile. There is no key in this file, no key in the request
|
|
JSON, no key in the response JSON, and no key in anything printed. Nothing
|
|
here ever reads a key into a variable, so nothing here can leak one. A missing
|
|
credential is a NOT_RUN with that reason.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
# Haiku, because the ruling was Haiku: the work code cannot do goes to the
|
|
# small model, and what the small model cannot settle goes to a person. It is
|
|
# also the only sane choice per-sheet on a model with four figures of them.
|
|
DEFAULT_MODEL = "claude-haiku-4-5"
|
|
|
|
# The three answers the skill allows. Anything else the model says is not an
|
|
# answer, and is treated as a refusal rather than coerced into one.
|
|
ANSWERS = ("clear", "problem_seen", "cannot_tell")
|
|
|
|
# One image, one question, one JSON object back. 400 output tokens is roughly
|
|
# four times the longest legal answer -- enough that a well-formed reply is
|
|
# never truncated, small enough that a runaway one costs nothing.
|
|
MAX_TOKENS = 400
|
|
|
|
# The API downscales any image whose long edge exceeds 1568 px, so a bigger
|
|
# render buys nothing and costs upload time. The extension renders to 1500;
|
|
# this is the ceiling it is staying under, restated here because the two
|
|
# halves cannot import each other and a silent drift between them would show
|
|
# up as unexplained illegibility.
|
|
IMAGE_LONG_EDGE_CEILING_PX = 1568
|
|
|
|
|
|
def _not_run(item, reason):
|
|
return {
|
|
"id": item.get("id"),
|
|
"question_id": item.get("question_id"),
|
|
"status": "NOT_RUN",
|
|
"reason": reason,
|
|
"answer": None,
|
|
"where": None,
|
|
"why": None,
|
|
"source": "model-read",
|
|
}
|
|
|
|
|
|
def _read_json(path):
|
|
with open(path, encoding="utf-8") as fh:
|
|
return json.load(fh)
|
|
|
|
|
|
def _write_json(path, obj):
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
json.dump(obj, fh, indent=2, sort_keys=True)
|
|
|
|
|
|
def load_skill(path):
|
|
"""The instructions the model reads, from the file that is versioned with
|
|
the code. Deliberately not a string in this module: a prompt buried in a
|
|
function is a prompt nobody reviews."""
|
|
with open(path, encoding="utf-8") as fh:
|
|
return fh.read()
|
|
|
|
|
|
def parse_answer(text, question_id):
|
|
"""The model's reply -> a result row, or None when it is not an answer.
|
|
|
|
Strict on purpose. A reply that is prose, or JSON with an answer word the
|
|
skill does not define, is NOT coerced into the nearest legal value -- it
|
|
is a refusal, because a coerced verdict is exactly the hallucination this
|
|
whole path is built to keep out of the report.
|
|
"""
|
|
if not text:
|
|
return None
|
|
body = text.strip()
|
|
if body.startswith("```"):
|
|
body = body.split("\n", 1)[-1]
|
|
if body.rstrip().endswith("```"):
|
|
body = body.rstrip()[:-3]
|
|
start, end = body.find("{"), body.rfind("}")
|
|
if start < 0 or end <= start:
|
|
return None
|
|
try:
|
|
obj = json.loads(body[start:end + 1])
|
|
except ValueError:
|
|
return None
|
|
if not isinstance(obj, dict):
|
|
return None
|
|
answer = obj.get("answer")
|
|
if answer not in ANSWERS:
|
|
return None
|
|
if obj.get("question_id") not in (None, question_id):
|
|
# It answered a different question than the one it was asked. That is
|
|
# not a near-miss to be salvaged; it means the reply is not about this
|
|
# image.
|
|
return None
|
|
why = obj.get("why")
|
|
where = obj.get("where")
|
|
return {
|
|
"answer": answer,
|
|
"where": where if isinstance(where, str) and where.strip() else None,
|
|
"why": why if isinstance(why, str) and why.strip() else None,
|
|
}
|
|
|
|
|
|
def build_user_content(item):
|
|
"""One image block, then the question. Image first: the guidance is that a
|
|
question about an image reads better after the image."""
|
|
with open(item["image"], "rb") as fh:
|
|
data = base64.standard_b64encode(fh.read()).decode("utf-8")
|
|
lines = [
|
|
"question_id: " + str(item.get("question_id")),
|
|
"",
|
|
"This is one view rendered off a precast shop ticket.",
|
|
]
|
|
if item.get("view_kind"):
|
|
lines.append("The view is a " + str(item["view_kind"]) + ".")
|
|
if item.get("effective_dpi"):
|
|
lines.append(
|
|
"It was rendered at about {0:.0f} dots per inch of the printed "
|
|
"sheet. If that was not enough to read it, say so."
|
|
.format(float(item["effective_dpi"])))
|
|
lines.append("")
|
|
lines.append("Answer the question named above, as one JSON object.")
|
|
return [
|
|
{"type": "image",
|
|
"source": {"type": "base64", "media_type": "image/png",
|
|
"data": data}},
|
|
{"type": "text", "text": "\n".join(lines)},
|
|
]
|
|
|
|
|
|
def ask(client, model, skill_text, item):
|
|
"""One call. Returns a result row; never raises for an API problem."""
|
|
try:
|
|
content = build_user_content(item)
|
|
except OSError as ex:
|
|
return _not_run(item, "the rendered image could not be read: "
|
|
"{0}".format(type(ex).__name__))
|
|
|
|
started = time.time()
|
|
try:
|
|
message = client.messages.create(
|
|
model=model,
|
|
max_tokens=MAX_TOKENS,
|
|
system=skill_text,
|
|
messages=[{"role": "user", "content": content}],
|
|
)
|
|
except Exception as ex: # noqa: BLE001 - see below
|
|
# Every failure to REACH the model is one outcome: NOT RUN, with the
|
|
# reason. Never a pass. The exception type is reported and the message
|
|
# is not, because an SDK error string can carry request detail and
|
|
# this file's whole discipline is that nothing sensitive is written
|
|
# to a file the tool keeps.
|
|
return _not_run(item, "the model could not be reached: {0}"
|
|
.format(type(ex).__name__))
|
|
|
|
text = "".join([b.text for b in message.content
|
|
if getattr(b, "type", None) == "text"])
|
|
parsed = parse_answer(text, item.get("question_id"))
|
|
if parsed is None:
|
|
return _not_run(item, "the model's reply was not one of the three "
|
|
"answers the skill defines")
|
|
|
|
usage = getattr(message, "usage", None)
|
|
row = _not_run(item, None)
|
|
row.update(parsed)
|
|
row["status"] = "OK"
|
|
row["reason"] = None
|
|
row["model"] = model
|
|
row["elapsed_s"] = round(time.time() - started, 2)
|
|
row["input_tokens"] = getattr(usage, "input_tokens", None)
|
|
row["output_tokens"] = getattr(usage, "output_tokens", None)
|
|
return row
|
|
|
|
|
|
def run(request):
|
|
"""The request dict -> the response dict. No I/O beyond the images."""
|
|
items = request.get("items") or []
|
|
model = request.get("model") or DEFAULT_MODEL
|
|
out = {"model": model, "source": "model-read", "results": []}
|
|
|
|
max_calls = request.get("max_calls")
|
|
if isinstance(max_calls, int) and max_calls >= 0 and len(items) > max_calls:
|
|
# The bound is the caller's, and it is enforced here as well as there.
|
|
# Over-budget items are NOT RUN naming the bound -- a dropped item
|
|
# would read as an item that passed.
|
|
for item in items[max_calls:]:
|
|
out["results"].append(_not_run(
|
|
item, "this run's vision call budget ({0}) was already spent"
|
|
.format(max_calls)))
|
|
items = items[:max_calls]
|
|
|
|
try:
|
|
skill_text = load_skill(request["skill"])
|
|
except (OSError, KeyError) as ex:
|
|
out["results"][:0] = [_not_run(
|
|
item, "the ticketqc-vision skill file could not be read "
|
|
"({0})".format(type(ex).__name__)) for item in items]
|
|
return out
|
|
|
|
try:
|
|
import anthropic
|
|
except ImportError:
|
|
out["results"][:0] = [_not_run(
|
|
item, "the anthropic SDK is not installed for this Python -- "
|
|
"pip install anthropic") for item in items]
|
|
return out
|
|
|
|
try:
|
|
client = anthropic.Anthropic()
|
|
except Exception as ex: # noqa: BLE001
|
|
out["results"][:0] = [_not_run(
|
|
item, "no Anthropic credential is available ({0}) -- set "
|
|
"ANTHROPIC_API_KEY in the environment, or run "
|
|
"`ant auth login`".format(type(ex).__name__))
|
|
for item in items]
|
|
return out
|
|
|
|
for i, item in enumerate(items):
|
|
out["results"].insert(i, ask(client, model, skill_text, item))
|
|
return out
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("request", help="request JSON written by ticketvision")
|
|
ap.add_argument("response", help="where to write the response JSON")
|
|
args = ap.parse_args(argv)
|
|
|
|
request = _read_json(args.request)
|
|
response = run(request)
|
|
_write_json(args.response, response)
|
|
|
|
ok = len([r for r in response["results"] if r["status"] == "OK"])
|
|
print("{0} answered, {1} NOT RUN".format(
|
|
ok, len(response["results"]) - ok))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|