#!/usr/bin/env python3 """Start one or more xcodebuild runs together and record what happened. usage: run.py LABEL REP SPEC [SPEC ...] SPEC is WORKTREE:DERIVED_DATA[@START_S][!keep] WORKTREE directory the build runs in (one git worktree per simulated agent) DERIVED_DATA value for -derivedDataPath; deleted first (clean build) unless the SPEC ends in !keep @START_S start this build START_S seconds after the batch starts (default 0) Environment: SCHEME default XcbLab DESTINATION default platform=macOS,arch=arm64 GATE_IDLE before starting, wait (up to 15 min) until `top` shows at least this % idle CPU; default 0 = off MAX_OTHER reject the batch if processes outside the builds averaged more than this summed %CPU; default off Every second a sampler reads `ps` and attributes processes to the experiment by process tree: the launched xcodebuild processes and all their descendants. It records swift-frontend counts, summed %CPU and RSS (MiB) of that tree, summed %CPU of everything else, the load average, and vm_stat pageouts/swapouts before and after the batch. %CPU from ps is a decaying average, not an exact core count. Output: one JSON line per batch in results/runs.jsonl (results/rejected.jsonl if rejected), a log per build in logs/, per-second samples in results/samples/. Exit status: 0 recorded, 3 rejected, 2 usage error. """ import json import os import re import shutil import subprocess import sys import threading import time HERE = os.path.dirname(os.path.abspath(__file__)) if len(sys.argv) < 4: print(__doc__, file=sys.stderr) sys.exit(2) label, rep = sys.argv[1], int(sys.argv[2]) specs = [] for arg in sys.argv[3:]: keep = arg.endswith("!keep") arg = arg[:-5] if keep else arg offset = 0.0 if "@" in arg: arg, o = arg.rsplit("@", 1) offset = float(o) worktree, dd = arg.split(":", 1) specs.append({"wt": os.path.abspath(worktree), "dd": os.path.abspath(dd), "offset": offset, "keep": keep}) scheme = os.environ.get("SCHEME", "XcbLab") destination = os.environ.get("DESTINATION", "platform=macOS,arch=arm64") gate = float(os.environ.get("GATE_IDLE", "0")) max_other = float(os.environ.get("MAX_OTHER", "inf")) for d in ("results/samples", "logs"): os.makedirs(os.path.join(HERE, d), exist_ok=True) def idle_pct(): out = subprocess.run(["top", "-l", "2", "-s", "2", "-n", "0"], capture_output=True, text=True).stdout vals = re.findall(r"([\d.]+)% idle", out) return float(vals[-1]) if vals else 0.0 def vm_counts(): out = subprocess.run(["vm_stat"], capture_output=True, text=True).stdout get = lambda k: int(re.search(rf"{k}:\s+(\d+)", out).group(1)) return {"pageouts": get("Pageouts"), "swapouts": get("Swapouts")} roots = set() samples = [] stop = threading.Event() def sample(t0): while not stop.is_set(): rows = [] for line in subprocess.run(["ps", "-A", "-o", "pid=,ppid=,pcpu=,rss=,comm="], capture_output=True, text=True).stdout.splitlines(): p = line.split(None, 4) if len(p) == 5: rows.append((int(p[0]), int(p[1]), float(p[2]), int(p[3]), p[4])) children = {} for pid, ppid, *_ in rows: children.setdefault(ppid, []).append(pid) tree, todo = set(), list(roots) while todo: pid = todo.pop() if pid not in tree: tree.add(pid) todo.extend(children.get(pid, [])) me = os.getpid() s = {"t": round(time.time() - t0, 1), "frontends": 0, "frontends_outside": 0, "build_cpu": 0.0, "build_rss_mib": 0, "other_cpu": 0.0, "load1": round(os.getloadavg()[0], 2)} for pid, ppid, cpu, rss, comm in rows: fe = comm.endswith("swift-frontend") if pid in tree: s["build_cpu"] += cpu s["build_rss_mib"] += rss // 1024 s["frontends"] += fe elif pid != me and ppid != me: s["other_cpu"] += cpu s["frontends_outside"] += fe samples.append(s) stop.wait(1.0) for s in specs: if not s["keep"]: shutil.rmtree(s["dd"], ignore_errors=True) waited, pre_idle = 0, idle_pct() while gate and pre_idle < gate and waited < 900: time.sleep(8) waited += 10 pre_idle = idle_pct() vm_before = vm_counts() t0 = time.time() sampler = threading.Thread(target=sample, args=(t0,), daemon=True) sampler.start() for idx, s in sorted(enumerate(specs), key=lambda e: e[1]["offset"]): delay = s["offset"] - (time.time() - t0) if delay > 0: time.sleep(delay) s["log"] = open(os.path.join(HERE, "logs", f"{label}-r{rep}-b{idx + 1}.log"), "w") cmd = ["xcodebuild", "-scheme", scheme, "-destination", destination, "-derivedDataPath", s["dd"], "build"] s["start"] = time.time() s["proc"] = subprocess.Popen(cmd, cwd=s["wt"], stdout=s["log"], stderr=subprocess.STDOUT) roots.add(s["proc"].pid) while any("end" not in s for s in specs): for s in specs: if "end" not in s and s["proc"].poll() is not None: s["end"] = time.time() s["log"].close() time.sleep(0.05) stop.set() sampler.join() vm_after = vm_counts() body = samples[1:] or samples mean = lambda k: round(sum(x[k] for x in body) / max(len(body), 1)) rec = { "label": label, "rep": rep, "n": len(specs), "started": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t0)), "builds": [{"index": i + 1, "worktree": os.path.basename(s["wt"]), "incremental": s["keep"], "start_offset_s": round(s["start"] - t0, 2), "wall_s": round(s["end"] - s["start"], 2), "exit": s["proc"].returncode} for i, s in enumerate(specs)], "batch_wall_s": round(max(s["end"] for s in specs) - t0, 2), "peak_frontends": max((x["frontends"] for x in samples), default=0), "peak_frontends_outside_tree": max((x["frontends_outside"] for x in samples), default=0), "peak_build_rss_mib": max((x["build_rss_mib"] for x in samples), default=0), "mean_build_cpu_pct": mean("build_cpu"), "mean_other_cpu_pct": mean("other_cpu"), "peak_load1": max((x["load1"] for x in samples), default=0), "pre_idle_pct": pre_idle, "gate_wait_s": waited, "pageouts": vm_after["pageouts"] - vm_before["pageouts"], "swapouts": vm_after["swapouts"] - vm_before["swapouts"], } rejected = rec["mean_other_cpu_pct"] > max_other with open(os.path.join(HERE, "results", "rejected.jsonl" if rejected else "runs.jsonl"), "a") as f: f.write(json.dumps(rec) + "\n") with open(os.path.join(HERE, "results", "samples", f"{label}-r{rep}.json"), "w") as f: json.dump(samples, f) print(json.dumps(rec)) sys.exit(3 if rejected else 0)