345 lines
14 KiB
Python
345 lines
14 KiB
Python
"""Exercise a built mio binary against local streaming model endpoints."""
|
|
|
|
import argparse
|
|
import contextlib
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import queue
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import threading
|
|
|
|
TEXT = "mio smoke passed"
|
|
MODEL = "mio-smoke-model"
|
|
KEY = "mio-test-credential"
|
|
ROUTES = {
|
|
"chat_completions": "/v1/chat/completions",
|
|
"responses": "/v1/responses",
|
|
"messages": "/v1/messages",
|
|
}
|
|
|
|
|
|
def sse(frames, named=False):
|
|
parts = []
|
|
for frame in frames:
|
|
if named:
|
|
parts.append(f"event: {frame['type']}\n")
|
|
parts.append(f"data: {json.dumps(frame)}\n\n")
|
|
if not named:
|
|
parts.append("data: [DONE]\n\n")
|
|
return "".join(parts).encode()
|
|
|
|
|
|
def chat_completion():
|
|
base = dict(id="chatcmpl-test", object="chat.completion.chunk", created=1, model=MODEL)
|
|
return sse([
|
|
dict(base, choices=[dict(index=0, delta=dict(role="assistant", content=TEXT), finish_reason=None)]),
|
|
dict(base, choices=[dict(index=0, delta={}, finish_reason="stop")]),
|
|
])
|
|
|
|
|
|
def response():
|
|
usage = dict(
|
|
input_tokens=1, output_tokens=1, total_tokens=2,
|
|
input_tokens_details=dict(cached_tokens=0),
|
|
output_tokens_details=dict(reasoning_tokens=0),
|
|
)
|
|
assistant_message = dict(
|
|
id="msg_test", type="message", role="assistant", status="completed",
|
|
content=[dict(type="output_text", text=TEXT, annotations=[], logprobs=[])],
|
|
)
|
|
completion = dict(
|
|
id="resp_test", object="response", created_at=1, model=MODEL,
|
|
status="completed", output=[assistant_message], usage=usage,
|
|
)
|
|
return sse([
|
|
dict(type="response.created", sequence_number=0,
|
|
response=dict(completion, status="in_progress", output=[], usage=None)),
|
|
dict(type="response.output_text.delta", sequence_number=1, item_id="msg_test",
|
|
output_index=0, content_index=0, delta=TEXT, logprobs=[]),
|
|
dict(type="response.completed", sequence_number=2, response=completion),
|
|
], named=True)
|
|
|
|
|
|
def message():
|
|
return sse([
|
|
dict(type="message_start", message=dict(
|
|
id="msg_test", type="message", role="assistant", model=MODEL,
|
|
content=[], stop_reason=None, stop_sequence=None,
|
|
usage=dict(input_tokens=1, output_tokens=0),
|
|
)),
|
|
dict(type="content_block_start", index=0, content_block=dict(type="text", text="")),
|
|
dict(type="content_block_delta", index=0, delta=dict(type="text_delta", text=TEXT)),
|
|
dict(type="content_block_stop", index=0),
|
|
dict(type="message_delta", delta=dict(stop_reason="end_turn", stop_sequence=None),
|
|
usage=dict(output_tokens=1)),
|
|
dict(type="message_stop"),
|
|
], named=True)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def mock_server(redirect_to=None):
|
|
observed = queue.SimpleQueue()
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, _format, *args):
|
|
pass
|
|
|
|
def reply(self, status, body=b"", headers=None):
|
|
self.send_response(status)
|
|
for key, value in (headers or {}).items():
|
|
self.send_header(key, value)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def do_POST(self):
|
|
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
|
observed.put(("POST", self.path, dict(self.headers), body))
|
|
routes = {ROUTES["chat_completions"]: chat_completion,
|
|
ROUTES["responses"]: response, ROUTES["messages"]: message}
|
|
if redirect_to and self.path == "/redirect/v1/chat/completions":
|
|
self.reply(307, headers={"Location": redirect_to})
|
|
elif self.path in routes:
|
|
self.reply(200, routes[self.path](), {"Content-Type": "text/event-stream"})
|
|
else:
|
|
self.reply(404)
|
|
|
|
def do_GET(self):
|
|
observed.put(("GET", self.path, dict(self.headers), b""))
|
|
self.reply(404)
|
|
|
|
def do_CONNECT(self):
|
|
observed.put(("CONNECT", self.path, dict(self.headers), b""))
|
|
self.reply(403)
|
|
|
|
with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server:
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
try:
|
|
yield f"http://127.0.0.1:{server.server_port}", observed
|
|
finally:
|
|
server.shutdown()
|
|
thread.join()
|
|
|
|
|
|
def read_stream(reader, received):
|
|
try:
|
|
while data := reader():
|
|
received.put(data)
|
|
except (EOFError, OSError):
|
|
pass
|
|
finally:
|
|
received.put(None)
|
|
|
|
|
|
def verify_acp(binary, root, env):
|
|
output = queue.SimpleQueue()
|
|
with subprocess.Popen([binary, "agent", "--no-leader", "stdio"],
|
|
cwd=root, env=env, stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) as process:
|
|
thread = threading.Thread(target=read_stream, args=(process.stdout.readline, output), daemon=True)
|
|
thread.start()
|
|
|
|
def call(request_id, method, params):
|
|
process.stdin.write(json.dumps(dict(jsonrpc="2.0", id=request_id,
|
|
method=method, params=params)) + "\n")
|
|
process.stdin.flush()
|
|
stop = time.monotonic() + 45
|
|
while (remaining := stop - time.monotonic()) > 0:
|
|
line = output.get(timeout=remaining)
|
|
assert line is not None, ("ACP exited", process.poll())
|
|
response = json.loads(line)
|
|
if response.get("id") == request_id:
|
|
assert "result" in response, response
|
|
return response["result"]
|
|
raise TimeoutError(f"ACP request timed out: {method}")
|
|
|
|
try:
|
|
initialized = call(1, "initialize", dict(protocolVersion=1, clientCapabilities={}))
|
|
assert not initialized.get("authMethods"), initialized
|
|
session = call(2, "session/new", dict(cwd=str(root), mcpServers=[]))
|
|
assert session.get("sessionId"), session
|
|
finally:
|
|
process.terminate()
|
|
process.wait(timeout=10)
|
|
thread.join(timeout=3)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def terminal(binary, root, env):
|
|
args = [binary, "--no-leader", "Respond with the test phrase."]
|
|
if os.name == "nt":
|
|
from winpty import PtyProcess
|
|
process = PtyProcess.spawn(args, cwd=str(root), env=env, dimensions=(32, 110))
|
|
try:
|
|
yield lambda: process.read(16384)
|
|
finally:
|
|
if process.isalive():
|
|
with contextlib.suppress(EOFError):
|
|
process.write("/exit\r")
|
|
deadline = time.monotonic() + 5
|
|
while process.isalive() and time.monotonic() < deadline:
|
|
time.sleep(0.05)
|
|
try:
|
|
process.close(force=True)
|
|
except PermissionError:
|
|
# ConPTY can exit between pywinpty's process check and its termination call.
|
|
if process.isalive():
|
|
raise
|
|
else:
|
|
import fcntl
|
|
import pty
|
|
import struct
|
|
import termios
|
|
|
|
master, slave = pty.openpty()
|
|
fcntl.ioctl(slave, termios.TIOCSWINSZ, struct.pack("HHHH", 32, 110, 0, 0))
|
|
try:
|
|
with subprocess.Popen(args, cwd=root, env=env, stdin=slave, stdout=slave,
|
|
stderr=slave, start_new_session=True) as process:
|
|
try:
|
|
yield lambda: os.read(master, 16384).decode("utf-8", "replace")
|
|
finally:
|
|
process.terminate()
|
|
try:
|
|
process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
finally:
|
|
os.close(slave)
|
|
os.close(master)
|
|
|
|
|
|
def verify_tui(binary, root, env):
|
|
output = queue.SimpleQueue()
|
|
screen = ""
|
|
with terminal(binary, root, env) as reader:
|
|
threading.Thread(target=read_stream, args=(reader, output), daemon=True).start()
|
|
stop = time.monotonic() + 45
|
|
while (remaining := stop - time.monotonic()) > 0:
|
|
try:
|
|
data = output.get(timeout=remaining)
|
|
except queue.Empty:
|
|
break
|
|
assert data is not None, screen
|
|
screen = (screen + data)[-64000:]
|
|
assert "preferred_method=api_key" not in screen and "Login with mio.invalid" not in screen, screen
|
|
if TEXT in screen:
|
|
return
|
|
raise TimeoutError(f"TUI did not display a model response: {screen}")
|
|
|
|
|
|
def verify(binary, tui):
|
|
with (
|
|
tempfile.TemporaryDirectory(prefix="mio-smoke-") as directory,
|
|
mock_server() as (proxy, unexpected),
|
|
mock_server(redirect_to=proxy + ROUTES["chat_completions"]) as (api, requests),
|
|
):
|
|
root = Path(directory)
|
|
settings = root / "config.toml"
|
|
env = {key: value for key, value in os.environ.items()
|
|
if not key.upper().startswith(("MIO_", "HOSHIZUKIMIO_", "OTEL_"))
|
|
and key.upper() not in {
|
|
"ALL_PROXY", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY",
|
|
"HOME", "USERPROFILE", "CODEX_HOME", "CLAUDE_CONFIG_DIR",
|
|
"XDG_CONFIG_HOME", "XDG_DATA_HOME",
|
|
}}
|
|
fake_home = root / "home"
|
|
fake_home.mkdir()
|
|
env.update(
|
|
HOME=str(fake_home), USERPROFILE=str(fake_home),
|
|
XDG_CONFIG_HOME=str(fake_home / "config"), XDG_DATA_HOME=str(fake_home / "data"),
|
|
MIO_HOME=directory, MIO_SMOKE_KEY=KEY, TERM="xterm-256color",
|
|
HTTP_PROXY=proxy, HTTPS_PROXY=proxy, ALL_PROXY=proxy,
|
|
NO_PROXY="localhost,127.0.0.1",
|
|
)
|
|
|
|
def run(*arguments, success=True):
|
|
process = subprocess.run([binary, *arguments], cwd=root, env=env, stdin=subprocess.DEVNULL,
|
|
capture_output=True, text=True, encoding="utf-8",
|
|
errors="replace", timeout=90)
|
|
output = process.stdout + process.stderr
|
|
assert (process.returncode == 0) == success, (arguments, process.returncode, output)
|
|
return output
|
|
|
|
assert "mio-code" in run("--help")
|
|
assert "mio" in run("--version").lower()
|
|
assert "config.toml" in run("models")
|
|
assert "config.toml" in run("-p", "test", success=False)
|
|
assert requests.empty()
|
|
assert unexpected.empty()
|
|
|
|
for backend, keyed, inherited in [
|
|
("chat_completions", True, False),
|
|
("responses", True, True),
|
|
("messages", True, False),
|
|
("chat_completions", False, True),
|
|
]:
|
|
endpoint = f'base_url = "{api}/v1"\napi_backend = "{backend}"\n'
|
|
if backend == "messages":
|
|
endpoint += 'env_http_headers = { "x-api-key" = "MIO_SMOKE_KEY" }\n'
|
|
endpoint += 'extra_headers = { "anthropic-version" = "2023-06-01" }\n'
|
|
elif keyed:
|
|
endpoint += 'env_key = "MIO_SMOKE_KEY"\n'
|
|
model = f'[models]\ndefault = "smoke"\n[model.smoke]\nmodel = "{MODEL}"\nmax_retries = 0\n'
|
|
if inherited:
|
|
model += 'model_provider = "local"\n[model_providers.local]\n'
|
|
settings.write_text(model + endpoint, encoding="utf-8")
|
|
|
|
assert "smoke (default)" in run("models")
|
|
assert requests.empty(), "listing models contacted the model endpoint"
|
|
assert TEXT in run("-p", "Respond with the test phrase.")
|
|
saw_inference = False
|
|
while not requests.empty():
|
|
method, path, headers, body = requests.get_nowait()
|
|
headers = {key.lower(): value for key, value in headers.items()}
|
|
if (method, path) == ("GET", "/"):
|
|
assert "authorization" not in headers and "x-api-key" not in headers, headers
|
|
continue
|
|
assert (method, path) == ("POST", ROUTES[backend]), (backend, method, path)
|
|
saw_inference = True
|
|
if backend == "messages":
|
|
assert headers.get("x-api-key") == KEY, headers
|
|
elif keyed:
|
|
assert headers.get("authorization") == f"Bearer {KEY}", headers
|
|
else:
|
|
assert not headers.get("authorization") and not headers.get("x-api-key"), headers
|
|
actual_model = json.loads(body)["model"]
|
|
assert actual_model == MODEL, (backend, method, path, actual_model)
|
|
assert saw_inference, backend
|
|
assert unexpected.empty(), unexpected.get_nowait()
|
|
print(f"Passed {backend} {'with credentials' if keyed else 'without credentials'}")
|
|
|
|
verify_acp(binary, root, env)
|
|
print("Passed ACP initialization and session creation")
|
|
if tui:
|
|
verify_tui(binary, root, env)
|
|
while not requests.empty():
|
|
requests.get_nowait()
|
|
assert unexpected.empty(), unexpected.get_nowait()
|
|
print("Passed interactive TUI inference")
|
|
|
|
settings.write_text(
|
|
f'[model.smoke]\nmodel = "{MODEL}"\nbase_url = "{api}/redirect/v1"\n'
|
|
'env_key = "MIO_SMOKE_KEY"\nmax_retries = 0\n', encoding="utf-8",
|
|
)
|
|
run("-p", "check redirect", success=False)
|
|
sent = []
|
|
while not requests.empty():
|
|
sent.append(requests.get_nowait()[:2])
|
|
assert ("POST", "/redirect" + ROUTES["chat_completions"]) in sent, sent
|
|
assert unexpected.empty(), unexpected.get_nowait()
|
|
print("Verified inference does not follow an HTTP redirect")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("binary", type=lambda path: str(Path(path).resolve(strict=True)))
|
|
parser.add_argument("--tui", action="store_true", help="also verify the interactive UI (pywinpty on Windows)")
|
|
options = parser.parse_args()
|
|
verify(options.binary, options.tui)
|