Files
mio-code/tests/smoke.py
2026-11-17 19:39:55 +08:00

345 lines
14 KiB
Python

"""Exercise a built mio binary against local streaming model endpoints."""
import argparse
import contextlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import queue
import subprocess
import tempfile
import time
import threading
TEXT = "mio smoke passed"
MODEL = "mio-smoke-model"
KEY = "mio-test-credential"
ROUTES = {
"chat_completions": "/v1/chat/completions",
"responses": "/v1/responses",
"messages": "/v1/messages",
}
def sse(frames, named=False):
parts = []
for frame in frames:
if named:
parts.append(f"event: {frame['type']}\n")
parts.append(f"data: {json.dumps(frame)}\n\n")
if not named:
parts.append("data: [DONE]\n\n")
return "".join(parts).encode()
def chat_completion():
base = dict(id="chatcmpl-test", object="chat.completion.chunk", created=1, model=MODEL)
return sse([
dict(base, choices=[dict(index=0, delta=dict(role="assistant", content=TEXT), finish_reason=None)]),
dict(base, choices=[dict(index=0, delta={}, finish_reason="stop")]),
])
def response():
usage = dict(
input_tokens=1, output_tokens=1, total_tokens=2,
input_tokens_details=dict(cached_tokens=0),
output_tokens_details=dict(reasoning_tokens=0),
)
assistant_message = dict(
id="msg_test", type="message", role="assistant", status="completed",
content=[dict(type="output_text", text=TEXT, annotations=[], logprobs=[])],
)
completion = dict(
id="resp_test", object="response", created_at=1, model=MODEL,
status="completed", output=[assistant_message], usage=usage,
)
return sse([
dict(type="response.created", sequence_number=0,
response=dict(completion, status="in_progress", output=[], usage=None)),
dict(type="response.output_text.delta", sequence_number=1, item_id="msg_test",
output_index=0, content_index=0, delta=TEXT, logprobs=[]),
dict(type="response.completed", sequence_number=2, response=completion),
], named=True)
def message():
return sse([
dict(type="message_start", message=dict(
id="msg_test", type="message", role="assistant", model=MODEL,
content=[], stop_reason=None, stop_sequence=None,
usage=dict(input_tokens=1, output_tokens=0),
)),
dict(type="content_block_start", index=0, content_block=dict(type="text", text="")),
dict(type="content_block_delta", index=0, delta=dict(type="text_delta", text=TEXT)),
dict(type="content_block_stop", index=0),
dict(type="message_delta", delta=dict(stop_reason="end_turn", stop_sequence=None),
usage=dict(output_tokens=1)),
dict(type="message_stop"),
], named=True)
@contextlib.contextmanager
def mock_server(redirect_to=None):
observed = queue.SimpleQueue()
class Handler(BaseHTTPRequestHandler):
def log_message(self, _format, *args):
pass
def reply(self, status, body=b"", headers=None):
self.send_response(status)
for key, value in (headers or {}).items():
self.send_header(key, value)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_POST(self):
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
observed.put(("POST", self.path, dict(self.headers), body))
routes = {ROUTES["chat_completions"]: chat_completion,
ROUTES["responses"]: response, ROUTES["messages"]: message}
if redirect_to and self.path == "/redirect/v1/chat/completions":
self.reply(307, headers={"Location": redirect_to})
elif self.path in routes:
self.reply(200, routes[self.path](), {"Content-Type": "text/event-stream"})
else:
self.reply(404)
def do_GET(self):
observed.put(("GET", self.path, dict(self.headers), b""))
self.reply(404)
def do_CONNECT(self):
observed.put(("CONNECT", self.path, dict(self.headers), b""))
self.reply(403)
with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server:
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
yield f"http://127.0.0.1:{server.server_port}", observed
finally:
server.shutdown()
thread.join()
def read_stream(reader, received):
try:
while data := reader():
received.put(data)
except (EOFError, OSError):
pass
finally:
received.put(None)
def verify_acp(binary, root, env):
output = queue.SimpleQueue()
with subprocess.Popen([binary, "agent", "--no-leader", "stdio"],
cwd=root, env=env, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) as process:
thread = threading.Thread(target=read_stream, args=(process.stdout.readline, output), daemon=True)
thread.start()
def call(request_id, method, params):
process.stdin.write(json.dumps(dict(jsonrpc="2.0", id=request_id,
method=method, params=params)) + "\n")
process.stdin.flush()
stop = time.monotonic() + 45
while (remaining := stop - time.monotonic()) > 0:
line = output.get(timeout=remaining)
assert line is not None, ("ACP exited", process.poll())
response = json.loads(line)
if response.get("id") == request_id:
assert "result" in response, response
return response["result"]
raise TimeoutError(f"ACP request timed out: {method}")
try:
initialized = call(1, "initialize", dict(protocolVersion=1, clientCapabilities={}))
assert not initialized.get("authMethods"), initialized
session = call(2, "session/new", dict(cwd=str(root), mcpServers=[]))
assert session.get("sessionId"), session
finally:
process.terminate()
process.wait(timeout=10)
thread.join(timeout=3)
@contextlib.contextmanager
def terminal(binary, root, env):
args = [binary, "--no-leader", "Respond with the test phrase."]
if os.name == "nt":
from winpty import PtyProcess
process = PtyProcess.spawn(args, cwd=str(root), env=env, dimensions=(32, 110))
try:
yield lambda: process.read(16384)
finally:
if process.isalive():
with contextlib.suppress(EOFError):
process.write("/exit\r")
deadline = time.monotonic() + 5
while process.isalive() and time.monotonic() < deadline:
time.sleep(0.05)
try:
process.close(force=True)
except PermissionError:
# ConPTY can exit between pywinpty's process check and its termination call.
if process.isalive():
raise
else:
import fcntl
import pty
import struct
import termios
master, slave = pty.openpty()
fcntl.ioctl(slave, termios.TIOCSWINSZ, struct.pack("HHHH", 32, 110, 0, 0))
try:
with subprocess.Popen(args, cwd=root, env=env, stdin=slave, stdout=slave,
stderr=slave, start_new_session=True) as process:
try:
yield lambda: os.read(master, 16384).decode("utf-8", "replace")
finally:
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
finally:
os.close(slave)
os.close(master)
def verify_tui(binary, root, env):
output = queue.SimpleQueue()
screen = ""
with terminal(binary, root, env) as reader:
threading.Thread(target=read_stream, args=(reader, output), daemon=True).start()
stop = time.monotonic() + 45
while (remaining := stop - time.monotonic()) > 0:
try:
data = output.get(timeout=remaining)
except queue.Empty:
break
assert data is not None, screen
screen = (screen + data)[-64000:]
assert "preferred_method=api_key" not in screen and "Login with mio.invalid" not in screen, screen
if TEXT in screen:
return
raise TimeoutError(f"TUI did not display a model response: {screen}")
def verify(binary, tui):
with (
tempfile.TemporaryDirectory(prefix="mio-smoke-") as directory,
mock_server() as (proxy, unexpected),
mock_server(redirect_to=proxy + ROUTES["chat_completions"]) as (api, requests),
):
root = Path(directory)
settings = root / "config.toml"
env = {key: value for key, value in os.environ.items()
if not key.upper().startswith(("MIO_", "HOSHIZUKIMIO_", "OTEL_"))
and key.upper() not in {
"ALL_PROXY", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY",
"HOME", "USERPROFILE", "CODEX_HOME", "CLAUDE_CONFIG_DIR",
"XDG_CONFIG_HOME", "XDG_DATA_HOME",
}}
fake_home = root / "home"
fake_home.mkdir()
env.update(
HOME=str(fake_home), USERPROFILE=str(fake_home),
XDG_CONFIG_HOME=str(fake_home / "config"), XDG_DATA_HOME=str(fake_home / "data"),
MIO_HOME=directory, MIO_SMOKE_KEY=KEY, TERM="xterm-256color",
HTTP_PROXY=proxy, HTTPS_PROXY=proxy, ALL_PROXY=proxy,
NO_PROXY="localhost,127.0.0.1",
)
def run(*arguments, success=True):
process = subprocess.run([binary, *arguments], cwd=root, env=env, stdin=subprocess.DEVNULL,
capture_output=True, text=True, encoding="utf-8",
errors="replace", timeout=90)
output = process.stdout + process.stderr
assert (process.returncode == 0) == success, (arguments, process.returncode, output)
return output
assert "mio-code" in run("--help")
assert "mio" in run("--version").lower()
assert "config.toml" in run("models")
assert "config.toml" in run("-p", "test", success=False)
assert requests.empty()
assert unexpected.empty()
for backend, keyed, inherited in [
("chat_completions", True, False),
("responses", True, True),
("messages", True, False),
("chat_completions", False, True),
]:
endpoint = f'base_url = "{api}/v1"\napi_backend = "{backend}"\n'
if backend == "messages":
endpoint += 'env_http_headers = { "x-api-key" = "MIO_SMOKE_KEY" }\n'
endpoint += 'extra_headers = { "anthropic-version" = "2023-06-01" }\n'
elif keyed:
endpoint += 'env_key = "MIO_SMOKE_KEY"\n'
model = f'[models]\ndefault = "smoke"\n[model.smoke]\nmodel = "{MODEL}"\nmax_retries = 0\n'
if inherited:
model += 'model_provider = "local"\n[model_providers.local]\n'
settings.write_text(model + endpoint, encoding="utf-8")
assert "smoke (default)" in run("models")
assert requests.empty(), "listing models contacted the model endpoint"
assert TEXT in run("-p", "Respond with the test phrase.")
saw_inference = False
while not requests.empty():
method, path, headers, body = requests.get_nowait()
headers = {key.lower(): value for key, value in headers.items()}
if (method, path) == ("GET", "/"):
assert "authorization" not in headers and "x-api-key" not in headers, headers
continue
assert (method, path) == ("POST", ROUTES[backend]), (backend, method, path)
saw_inference = True
if backend == "messages":
assert headers.get("x-api-key") == KEY, headers
elif keyed:
assert headers.get("authorization") == f"Bearer {KEY}", headers
else:
assert not headers.get("authorization") and not headers.get("x-api-key"), headers
actual_model = json.loads(body)["model"]
assert actual_model == MODEL, (backend, method, path, actual_model)
assert saw_inference, backend
assert unexpected.empty(), unexpected.get_nowait()
print(f"Passed {backend} {'with credentials' if keyed else 'without credentials'}")
verify_acp(binary, root, env)
print("Passed ACP initialization and session creation")
if tui:
verify_tui(binary, root, env)
while not requests.empty():
requests.get_nowait()
assert unexpected.empty(), unexpected.get_nowait()
print("Passed interactive TUI inference")
settings.write_text(
f'[model.smoke]\nmodel = "{MODEL}"\nbase_url = "{api}/redirect/v1"\n'
'env_key = "MIO_SMOKE_KEY"\nmax_retries = 0\n', encoding="utf-8",
)
run("-p", "check redirect", success=False)
sent = []
while not requests.empty():
sent.append(requests.get_nowait()[:2])
assert ("POST", "/redirect" + ROUTES["chat_completions"]) in sent, sent
assert unexpected.empty(), unexpected.get_nowait()
print("Verified inference does not follow an HTTP redirect")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("binary", type=lambda path: str(Path(path).resolve(strict=True)))
parser.add_argument("--tui", action="store_true", help="also verify the interactive UI (pywinpty on Windows)")
options = parser.parse_args()
verify(options.binary, options.tui)