will
52d94303bc
ci / test (push) Has been cancelled
Public demo snapshot of an in-house pyRevit extension for precast concrete shop-ticket QC (37 checks / 46 checklist rows) and a standalone panel handling solver (PCI Chapter 8). - No client, job, plant or product data: knowledge layers ship as structural templates only. - Ticket QC's model-read escalation targets a LAN model server endpoint (configurable), not a cloud provider. - Built from the private development repo via tools/make_dist.py.
288 lines
11 KiB
Python
288 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Ask a small vision model one question about one rendered ticket view.
|
|
|
|
python tools/vision_read.py <request.json> <response.json>
|
|
|
|
This is the CPython half of the Ticket QC vision escalation. It exists as a
|
|
separate process for the same two reasons the other out-of-process tools do:
|
|
|
|
* the extension runs on IronPython 2.7 inside Revit's UI thread, and there
|
|
is no HTTP client there worth trusting with a model call;
|
|
* nothing here touches the Revit API. It reads PNGs and JSON and writes
|
|
JSON, so it has no business being inside Revit.
|
|
|
|
The two halves talk through files, never imports. `vibe.ticketvision` builds
|
|
the request, this writes the response, and neither imports the other.
|
|
|
|
WHAT THIS MAY AND MAY NOT DO
|
|
----------------------------
|
|
It may return a verdict from a model that looked at a picture. It may not
|
|
return anything that renders like a computed result. Every row it writes
|
|
carries `source: "model-read"`, and a row it could not obtain is
|
|
`status: "NOT_RUN"` with a reason -- never a pass, never silence. Reaching
|
|
the model is not the same as the model being right, and neither is the same
|
|
as this program having run at all; the three are kept separate on purpose.
|
|
|
|
ENDPOINT
|
|
--------
|
|
The model server lives on the office LAN (default `http://192.168.99.151`,
|
|
override with `VISION_ENDPOINT` or an `endpoint` key in the request). The
|
|
endpoint speaks the OpenAI chat/completions wire format; the model name
|
|
(`will-tool-ai` by default) is whatever that server advertises. There is no
|
|
key in this file, no key in the request JSON, no key in the response JSON,
|
|
and no key in anything printed. Nothing here ever reads a key into a
|
|
variable, so nothing here can leak one. An unreachable server is a NOT_RUN
|
|
with that reason.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
# The LAN vision server this tool talks to. Nothing here calls a cloud
|
|
# provider: the endpoint is a local model server on the office network, and
|
|
# the model name is the one that server advertises. Override either with the
|
|
# environment when the server moves.
|
|
DEFAULT_MODEL = "will-tool-ai"
|
|
DEFAULT_ENDPOINT = "http://192.168.99.151/v1/chat/completions"
|
|
|
|
# The three answers the skill allows. Anything else the model says is not an
|
|
# answer, and is treated as a refusal rather than coerced into one.
|
|
ANSWERS = ("clear", "problem_seen", "cannot_tell")
|
|
|
|
# One image, one question, one JSON object back. 400 output tokens is roughly
|
|
# four times the longest legal answer -- enough that a well-formed reply is
|
|
# never truncated, small enough that a runaway one costs nothing.
|
|
MAX_TOKENS = 400
|
|
|
|
# The API downscales any image whose long edge exceeds 1568 px, so a bigger
|
|
# render buys nothing and costs upload time. The extension renders to 1500;
|
|
# this is the ceiling it is staying under, restated here because the two
|
|
# halves cannot import each other and a silent drift between them would show
|
|
# up as unexplained illegibility.
|
|
IMAGE_LONG_EDGE_CEILING_PX = 1568
|
|
|
|
|
|
def _not_run(item, reason):
|
|
return {
|
|
"id": item.get("id"),
|
|
"question_id": item.get("question_id"),
|
|
"status": "NOT_RUN",
|
|
"reason": reason,
|
|
"answer": None,
|
|
"where": None,
|
|
"why": None,
|
|
"source": "model-read",
|
|
}
|
|
|
|
|
|
def _read_json(path):
|
|
with open(path, encoding="utf-8") as fh:
|
|
return json.load(fh)
|
|
|
|
|
|
def _write_json(path, obj):
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
json.dump(obj, fh, indent=2, sort_keys=True)
|
|
|
|
|
|
def load_skill(path):
|
|
"""The instructions the model reads, from the file that is versioned with
|
|
the code. Deliberately not a string in this module: a prompt buried in a
|
|
function is a prompt nobody reviews."""
|
|
with open(path, encoding="utf-8") as fh:
|
|
return fh.read()
|
|
|
|
|
|
def parse_answer(text, question_id):
|
|
"""The model's reply -> a result row, or None when it is not an answer.
|
|
|
|
Strict on purpose. A reply that is prose, or JSON with an answer word the
|
|
skill does not define, is NOT coerced into the nearest legal value -- it
|
|
is a refusal, because a coerced verdict is exactly the hallucination this
|
|
whole path is built to keep out of the report.
|
|
"""
|
|
if not text:
|
|
return None
|
|
body = text.strip()
|
|
if body.startswith("```"):
|
|
body = body.split("\n", 1)[-1]
|
|
if body.rstrip().endswith("```"):
|
|
body = body.rstrip()[:-3]
|
|
start, end = body.find("{"), body.rfind("}")
|
|
if start < 0 or end <= start:
|
|
return None
|
|
try:
|
|
obj = json.loads(body[start:end + 1])
|
|
except ValueError:
|
|
return None
|
|
if not isinstance(obj, dict):
|
|
return None
|
|
answer = obj.get("answer")
|
|
if answer not in ANSWERS:
|
|
return None
|
|
if obj.get("question_id") not in (None, question_id):
|
|
# It answered a different question than the one it was asked. That is
|
|
# not a near-miss to be salvaged; it means the reply is not about this
|
|
# image.
|
|
return None
|
|
why = obj.get("why")
|
|
where = obj.get("where")
|
|
return {
|
|
"answer": answer,
|
|
"where": where if isinstance(where, str) and where.strip() else None,
|
|
"why": why if isinstance(why, str) and why.strip() else None,
|
|
}
|
|
|
|
|
|
def build_user_content(item):
|
|
"""One image block, then the question. Image first: the guidance is that a
|
|
question about an image reads better after the image."""
|
|
with open(item["image"], "rb") as fh:
|
|
data = base64.standard_b64encode(fh.read()).decode("utf-8")
|
|
lines = [
|
|
"question_id: " + str(item.get("question_id")),
|
|
"",
|
|
"This is one view rendered off a precast shop ticket.",
|
|
]
|
|
if item.get("view_kind"):
|
|
lines.append("The view is a " + str(item["view_kind"]) + ".")
|
|
if item.get("effective_dpi"):
|
|
lines.append(
|
|
"It was rendered at about {0:.0f} dots per inch of the printed "
|
|
"sheet. If that was not enough to read it, say so."
|
|
.format(float(item["effective_dpi"])))
|
|
lines.append("")
|
|
lines.append("Answer the question named above, as one JSON object.")
|
|
# OpenAI-compatible chat format: image_url carries the base64 data URI.
|
|
mime = "image/png"
|
|
if item.get("image", "").lower().endswith((".jpg", ".jpeg")):
|
|
mime = "image/jpeg"
|
|
return [
|
|
{"type": "image_url",
|
|
"image_url": {"url": "data:{0};base64,{1}".format(mime, data)}},
|
|
{"type": "text", "text": "\n".join(lines)},
|
|
]
|
|
|
|
|
|
def ask(endpoint, model, skill_text, item):
|
|
"""One call. Returns a result row; never raises for an API problem."""
|
|
try:
|
|
content = build_user_content(item)
|
|
except OSError as ex:
|
|
return _not_run(item, "the rendered image could not be read: "
|
|
"{0}".format(type(ex).__name__))
|
|
|
|
started = time.time()
|
|
try:
|
|
body = json.dumps({
|
|
"model": model,
|
|
"max_tokens": MAX_TOKENS,
|
|
"messages": [
|
|
{"role": "system", "content": skill_text},
|
|
{"role": "user", "content": content},
|
|
],
|
|
}).encode("utf-8")
|
|
req = urllib.request.Request(endpoint, data=body, headers={
|
|
"Content-Type": "application/json",
|
|
})
|
|
with urllib.request.urlopen(req, timeout=180) as resp:
|
|
reply = json.loads(resp.read().decode("utf-8"))
|
|
except Exception as ex: # noqa: BLE001 - see below
|
|
# Every failure to REACH the model is one outcome: NOT RUN, with the
|
|
# reason. Never a pass. The exception type is reported and the message
|
|
# is not, because an error string can carry request detail and
|
|
# this file's whole discipline is that nothing sensitive is written
|
|
# to a file the tool keeps.
|
|
return _not_run(item, "the model could not be reached: {0}"
|
|
.format(type(ex).__name__))
|
|
|
|
try:
|
|
text = "".join(
|
|
c.get("text", "") for c in reply["choices"][0]["message"]["content"]
|
|
if isinstance(c, dict) and c.get("type") == "text")
|
|
except (KeyError, IndexError, TypeError):
|
|
text = reply.get("choices", [{}])[0].get("message", {}).get("content", "")
|
|
if isinstance(text, str):
|
|
pass
|
|
else:
|
|
text = ""
|
|
parsed = parse_answer(text, item.get("question_id"))
|
|
if parsed is None:
|
|
return _not_run(item, "the model's reply was not one of the three "
|
|
"answers the skill defines")
|
|
|
|
usage = reply.get("usage", {})
|
|
row = _not_run(item, None)
|
|
row.update(parsed)
|
|
row["status"] = "OK"
|
|
row["reason"] = None
|
|
row["model"] = model
|
|
row["elapsed_s"] = round(time.time() - started, 2)
|
|
row["input_tokens"] = usage.get("prompt_tokens")
|
|
row["output_tokens"] = usage.get("completion_tokens")
|
|
return row
|
|
|
|
|
|
def run(request):
|
|
"""The request dict -> the response dict. No I/O beyond the images."""
|
|
items = request.get("items") or []
|
|
model = request.get("model") or DEFAULT_MODEL
|
|
out = {"model": model, "source": "model-read", "results": []}
|
|
|
|
max_calls = request.get("max_calls")
|
|
if isinstance(max_calls, int) and max_calls >= 0 and len(items) > max_calls:
|
|
# The bound is the caller's, and it is enforced here as well as there.
|
|
# Over-budget items are NOT RUN naming the bound -- a dropped item
|
|
# would read as an item that passed.
|
|
for item in items[max_calls:]:
|
|
out["results"].append(_not_run(
|
|
item, "this run's vision call budget ({0}) was already spent"
|
|
.format(max_calls)))
|
|
items = items[:max_calls]
|
|
|
|
try:
|
|
skill_text = load_skill(request["skill"])
|
|
except (OSError, KeyError) as ex:
|
|
out["results"][:0] = [_not_run(
|
|
item, "the ticketqc-vision skill file could not be read "
|
|
"({0})".format(type(ex).__name__)) for item in items]
|
|
return out
|
|
|
|
endpoint = (request.get("endpoint") or
|
|
os.environ.get("VISION_ENDPOINT") or DEFAULT_ENDPOINT)
|
|
# The endpoint is a bare host[:port] in some configs; accept both that
|
|
# and a full URL, and always hit the chat/completions path.
|
|
if not endpoint.startswith("http"):
|
|
endpoint = "http://{0}/v1/chat/completions".format(endpoint)
|
|
|
|
for i, item in enumerate(items):
|
|
out["results"].insert(i, ask(endpoint, model, skill_text, item))
|
|
return out
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("request", help="request JSON written by ticketvision")
|
|
ap.add_argument("response", help="where to write the response JSON")
|
|
args = ap.parse_args(argv)
|
|
|
|
request = _read_json(args.request)
|
|
response = run(request)
|
|
_write_json(args.response, response)
|
|
|
|
ok = len([r for r in response["results"] if r["status"] == "OK"])
|
|
print("{0} answered, {1} NOT RUN".format(
|
|
ok, len(response["results"]) - ok))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|