Martin Vogel
109 lines
4.2 KiB
Python
109 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Drive a fixed sequence of MCP requests over stdio and wait for each reply.
|
|
|
|
Batching all requests and closing stdin does not work: the server treats EOF as
|
|
the client disconnecting and shuts down before answering, so the run produces a
|
|
census of a process that never did any work. Reading each response before
|
|
sending the next also makes the workload deterministic — every request is
|
|
serviced from the same steady state, which is what makes two runs comparable.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
|
|
def rpc(proc, payload, timeout_note):
|
|
proc.stdin.write(json.dumps(payload) + "\n")
|
|
proc.stdin.flush()
|
|
while True:
|
|
line = proc.stdout.readline()
|
|
if not line:
|
|
raise RuntimeError(f"server closed stdout while waiting for {timeout_note}")
|
|
line = line.strip()
|
|
if not line.startswith("{"):
|
|
continue
|
|
try:
|
|
message = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if message.get("id") == payload.get("id"):
|
|
return message
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("binary")
|
|
parser.add_argument("corpus")
|
|
parser.add_argument("requests", type=int)
|
|
parser.add_argument("--stderr", help="file for the server's stderr; never discard it")
|
|
parser.add_argument("--tool", default="search_graph",
|
|
help="tool to repeat; varying it isolates which path leaks")
|
|
parser.add_argument("--idle-seconds", type=float, default=0.0,
|
|
help="pause midway; separates per-request growth from per-second growth")
|
|
parser.add_argument("--skip-index", action="store_true",
|
|
help="omit the initial index, to separate store setup from the loop")
|
|
args = parser.parse_args()
|
|
|
|
proc = subprocess.Popen(
|
|
[args.binary],
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
# Never DEVNULL: when the server refuses to start, its stderr is the
|
|
# only thing that says why.
|
|
stderr=open(args.stderr, "w") if args.stderr else None,
|
|
text=True,
|
|
bufsize=1,
|
|
)
|
|
served = 0
|
|
failures = 0
|
|
try:
|
|
rpc(proc, {"jsonrpc": "2.0", "id": 0, "method": "initialize",
|
|
"params": {"protocolVersion": "2024-11-05", "capabilities": {},
|
|
"clientInfo": {"name": "memlab", "version": "1"}}},
|
|
"initialize")
|
|
if not args.skip_index:
|
|
indexed = rpc(proc, {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
|
|
"params": {"name": "index_repository",
|
|
"arguments": {"path": args.corpus}}},
|
|
"index_repository")
|
|
if "error" in indexed:
|
|
print(f"index failed: {indexed['error']}", file=sys.stderr)
|
|
return 3
|
|
|
|
halfway = 2 + args.requests // 2
|
|
for i in range(2, args.requests + 2):
|
|
if args.idle_seconds > 0 and i == halfway:
|
|
# Nothing is requested during this gap. Any commit growth across
|
|
# it belongs to a background thread, not the request path.
|
|
print(f"idle-start id={i}", flush=True)
|
|
time.sleep(args.idle_seconds)
|
|
print(f"idle-end id={i}", flush=True)
|
|
arguments = {"search_graph": {"name_pattern": ".*Widget.*", "limit": 10},
|
|
"list_projects": {},
|
|
"get_graph_schema": {},
|
|
"search_code": {"pattern": "Widget"}}.get(args.tool, {})
|
|
reply = rpc(proc, {"jsonrpc": "2.0", "id": i, "method": "tools/call",
|
|
"params": {"name": args.tool, "arguments": arguments}},
|
|
f"request {i}")
|
|
served += 1
|
|
if "error" in reply:
|
|
failures += 1
|
|
finally:
|
|
try:
|
|
proc.stdin.close()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
proc.wait(timeout=30)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
|
|
print(f"served={served} failed={failures}")
|
|
return 0 if served == args.requests and failures == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|