#!/usr/bin/env python3 """Aggregates the hey outputs of load.sh into the tables of Приложение 3. load_analyze.py RESULTS_DIR RESULTS_DIR/manifest.csv rows: scenario,mode,label,clients,file mode par = the files of the scenario ran at the same time (duration = max) seq = they ran one after another (duration = sum) Each file is a default (summary) hey output. Per scenario the program reports requests, clients, duration, RPS, success %, status codes, transport errors, min/avg/P50/P90/P95/P99/max (ms). As in the protocol, percentiles of a scenario made of several hey processes/runs are averages weighted by the number of responses; min/max are exact. Criteria (Приложение 3, табл. 8): K1 success >= 99 %, K2 no 5xx, K3 P95 <= 2000 ms; K4 (S6 vs S1): /readyz 200 and S6 P50 <= 2 x S1 P50 + 1 ms. Writes summary.csv, summary.json, detail.csv into RESULTS_DIR and prints the tables. Exit code 1 when a criterion fails. """ import csv import json import os import re import sys PCTS = ["10", "25", "50", "75", "90", "95", "99"] def parse_hey(path): txt = open(path, encoding="utf-8", errors="replace").read() def num(label): m = re.search(r"^\s*%s:\s+([0-9.]+)" % re.escape(label), txt, re.M) return float(m.group(1)) if m else None r = {"total_s": num("Total"), "slowest_s": num("Slowest"), "fastest_s": num("Fastest"), "average_s": num("Average"), "rps": num("Requests/sec"), "codes": {}, "errors": 0, "pct": {}} for p, v in re.findall(r"^\s*(\d+)%+ in ([0-9.]+) secs", txt, re.M): if p != "0": # hey 0.1.5 prints "%%" and a bogus "0%% in 0.0000" row for tiny samples r["pct"][p] = float(v) sec = txt.split("Status code distribution:", 1) if len(sec) == 2: for code, n in re.findall(r"^\s*\[(\d+)\]\s+(\d+) responses", sec[1].split("Error distribution:")[0], re.M): r["codes"][code] = r["codes"].get(code, 0) + int(n) sec = txt.split("Error distribution:", 1) if len(sec) == 2: r["errors"] = sum(int(n) for n in re.findall(r"^\s*\[(\d+)\]\s", sec[1], re.M)) r["responses"] = sum(r["codes"].values()) r["requests"] = r["responses"] + r["errors"] return r def aggregate(rows, mode): parts = [x["hey"] for x in rows] resp = sum(p["responses"] for p in parts) req = sum(p["requests"] for p in parts) durs = [p["total_s"] or 0 for p in parts] dur = max(durs) if mode == "par" else sum(durs) codes = {} for p in parts: for c, n in p["codes"].items(): codes[c] = codes.get(c, 0) + n def wavg(get): num = sum(get(p) * p["responses"] for p in parts if get(p) is not None and p["responses"]) den = sum(p["responses"] for p in parts if get(p) is not None and p["responses"]) return num / den if den else None ms = lambda v: None if v is None else v * 1000.0 ok2xx = sum(n for c, n in codes.items() if c.startswith("2")) fast = [p["fastest_s"] for p in parts if p["fastest_s"] is not None] slow = [p["slowest_s"] for p in parts if p["slowest_s"] is not None] return { "requests": req, "responses": resp, "errors": sum(p["errors"] for p in parts), "clients": sum(int(x["clients"]) for x in rows) if mode == "par" else max(int(x["clients"]) for x in rows), "duration_s": dur, "rps": (resp / dur) if dur else None, "success_pct": (100.0 * ok2xx / req) if req else 0.0, "codes": codes, "n5xx": sum(n for c, n in codes.items() if c.startswith("5")), "min_ms": ms(min(fast)) if fast else None, "avg_ms": ms(wavg(lambda p: p["average_s"])), **{"p%s_ms" % k: ms(wavg(lambda p, k=k: p["pct"].get(k))) for k in ("50", "90", "95", "99")}, "max_ms": ms(max(slow)) if slow else None, } def f1(v, nd=1): return "-" if v is None else ("%.*f" % (nd, v)) def codes_str(c): return ", ".join("%s: %d" % (k, v) for k, v in sorted(c.items())) or "-" def main(): d = sys.argv[1] rows = [] with open(os.path.join(d, "manifest.csv"), encoding="utf-8") as f: for x in csv.DictReader(f): x["hey"] = parse_hey(os.path.join(d, "raw", x["file"])) rows.append(x) order, by = [], {} for x in rows: if x["scenario"] not in by: order.append(x["scenario"]) by[x["scenario"]] = [] by[x["scenario"]].append(x) summary = [] for s in order: agg = aggregate(by[s], by[s][0]["mode"]) agg["scenario"] = s agg["k1"] = agg["success_pct"] >= 99.0 agg["k2"] = agg["n5xx"] == 0 agg["k3"] = agg["p95_ms"] is not None and agg["p95_ms"] <= 2000.0 summary.append(agg) extra = {} xp = os.path.join(d, "extra.json") if os.path.exists(xp): extra = json.load(open(xp, encoding="utf-8")) k4 = None s1 = next((x for x in summary if x["scenario"] == "S1"), None) s6 = next((x for x in summary if x["scenario"] == "S6"), None) if s1 and s6: k4 = (extra.get("readyz_after") == 200 and s6["p50_ms"] is not None and s1["p50_ms"] is not None and s6["p50_ms"] <= 2 * s1["p50_ms"] + 1.0) hdr = ["Сцен.", "Запр.", "Клиентов", "Длит.,с", "RPS", "Успех,%", "min", "avg", "P50", "P90", "P95", "P99", "max", "Коды", "Ошибки", "Итог"] lines = [] for a in summary: ok = a["k1"] and a["k2"] and a["k3"] lines.append([a["scenario"], str(a["requests"]), str(a["clients"]), f1(a["duration_s"]), f1(a["rps"], 0), f1(a["success_pct"], 2), f1(a["min_ms"]), f1(a["avg_ms"]), f1(a["p50_ms"]), f1(a["p90_ms"]), f1(a["p95_ms"]), f1(a["p99_ms"]), f1(a["max_ms"]), codes_str(a["codes"]), str(a["errors"]), "PASS" if ok else "FAIL"]) print_table("Сводные результаты (время — мс)", hdr, lines) for s in order: if len(by[s]) > 1 and s not in ("S1", "S6"): dl = [] for x in by[s]: a = aggregate([x], "par") dl.append([x["label"], x["clients"], str(a["responses"]), f1(a["rps"], 1), f1(a["p50_ms"]), f1(a["p95_ms"]), f1(a["p99_ms"]), f1(a["max_ms"]), codes_str(a["codes"]), str(a["errors"])]) print_table("%s по эндпоинтам / прогонам (время — мс)" % s, ["Эндпоинт / прогон", "Клиентов", "Ответов", "RPS", "P50", "P95", "P99", "max", "Коды", "Ошибки"], dl) if extra.get("uploads"): u = extra["uploads"] print("S5 загрузки: %d запросов POST /datasets, коды %s, время ответа %s–%s мс; обработано до ready: %d/%d, макс. время обработки %s с" % (u["count"], u["codes"], f1(u.get("min_ms")), f1(u.get("max_ms")), u.get("ready", 0), u["count"], f1(u.get("max_processing_s"), 2))) if extra.get("stats_peak"): print("\nПиковое потребление ресурсов (docker stats, сценарии S4/S5):") for name, v in sorted(extra["stats_peak"].items()): print(" %-34s CPU %6.1f %% память %8.1f МиБ" % (name, v["cpu"], v["mem_mib"])) if k4 is not None: print("\nK4 (после нагрузки): /readyz %s; S6 P50/P95 = %s/%s мс против S1 %s/%s мс -> %s" % ( extra.get("readyz_after"), f1(s6["p50_ms"]), f1(s6["p95_ms"]), f1(s1["p50_ms"]), f1(s1["p95_ms"]), "PASS" if k4 else "FAIL")) with open(os.path.join(d, "summary.csv"), "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["scenario", "requests", "clients", "duration_s", "rps", "success_pct", "min_ms", "avg_ms", "p50_ms", "p90_ms", "p95_ms", "p99_ms", "max_ms", "codes", "errors", "result"]) for ln, a in zip(lines, summary): w.writerow([a["scenario"], a["requests"], a["clients"], a["duration_s"], a["rps"], a["success_pct"], a["min_ms"], a["avg_ms"], a["p50_ms"], a["p90_ms"], a["p95_ms"], a["p99_ms"], a["max_ms"], codes_str(a["codes"]), a["errors"], ln[-1]]) with open(os.path.join(d, "detail.csv"), "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["scenario", "label", "clients", "requests", "responses", "errors", "duration_s", "rps", "min_ms", "avg_ms", "p50_ms", "p90_ms", "p95_ms", "p99_ms", "max_ms", "codes", "file"]) for x in rows: a = aggregate([x], "par") w.writerow([x["scenario"], x["label"], x["clients"], a["requests"], a["responses"], a["errors"], a["duration_s"], a["rps"], a["min_ms"], a["avg_ms"], a["p50_ms"], a["p90_ms"], a["p95_ms"], a["p99_ms"], a["max_ms"], codes_str(a["codes"]), x["file"]]) with open(os.path.join(d, "summary.json"), "w", encoding="utf-8") as f: json.dump({"scenarios": summary, "k4": k4, "extra": extra}, f, ensure_ascii=False, indent=2) failed = [a["scenario"] for a in summary if not (a["k1"] and a["k2"] and a["k3"])] if k4 is False: failed.append("K4") print("\nКритерии K1 (успех >= 99 %%), K2 (5xx = 0), K3 (P95 <= 2000 мс)%s: %s" % ( ", K4" if k4 is not None else "", "все выполнены" if not failed else "НЕ выполнены: " + ", ".join(failed))) print("Файлы: %s/{summary.csv,summary.json,detail.csv,raw/}" % d) return 1 if failed else 0 def print_table(title, hdr, lines): widths = [max(len(str(r[i])) for r in [hdr] + lines) for i in range(len(hdr))] print("\n" + title) print(" | ".join(h.ljust(w) for h, w in zip(hdr, widths))) print("-+-".join("-" * w for w in widths)) for r in lines: print(" | ".join(str(c).ljust(w) for c, w in zip(r, widths))) if __name__ == "__main__": sys.exit(main())