gis/test/acceptance/lib/load_analyze.py

#!/usr/bin/env python3
"""Aggregates the hey outputs of load.sh into the tables of Приложение 3.

    load_analyze.py RESULTS_DIR

RESULTS_DIR/manifest.csv rows: scenario,mode,label,clients,file
  mode  par = the files of the scenario ran at the same time (duration = max)
        seq = they ran one after another (duration = sum)
Each file is a default (summary) hey output. Per scenario the program reports
requests, clients, duration, RPS, success %, status codes, transport errors,
min/avg/P50/P90/P95/P99/max (ms). As in the protocol, percentiles of a
scenario made of several hey processes/runs are averages weighted by the
number of responses; min/max are exact.

Criteria (Приложение 3, табл. 8): K1 success >= 99 %, K2 no 5xx, K3 P95 <=
2000 ms; K4 (S6 vs S1): /readyz 200 and S6 P50 <= 2 x S1 P50 + 1 ms.

Writes summary.csv, summary.json, detail.csv into RESULTS_DIR and prints the
tables. Exit code 1 when a criterion fails.
"""
import csv
import json
import os
import re
import sys

PCTS = ["10", "25", "50", "75", "90", "95", "99"]


def parse_hey(path):
    txt = open(path, encoding="utf-8", errors="replace").read()

    def num(label):
        m = re.search(r"^\s*%s:\s+([0-9.]+)" % re.escape(label), txt, re.M)
        return float(m.group(1)) if m else None

    r = {"total_s": num("Total"), "slowest_s": num("Slowest"), "fastest_s": num("Fastest"),
         "average_s": num("Average"), "rps": num("Requests/sec"), "codes": {}, "errors": 0, "pct": {}}
    for p, v in re.findall(r"^\s*(\d+)%+ in ([0-9.]+) secs", txt, re.M):
        if p != "0":  # hey 0.1.5 prints "%%" and a bogus "0%% in 0.0000" row for tiny samples
            r["pct"][p] = float(v)
    sec = txt.split("Status code distribution:", 1)
    if len(sec) == 2:
        for code, n in re.findall(r"^\s*\[(\d+)\]\s+(\d+) responses", sec[1].split("Error distribution:")[0], re.M):
            r["codes"][code] = r["codes"].get(code, 0) + int(n)
    sec = txt.split("Error distribution:", 1)
    if len(sec) == 2:
        r["errors"] = sum(int(n) for n in re.findall(r"^\s*\[(\d+)\]\s", sec[1], re.M))
    r["responses"] = sum(r["codes"].values())
    r["requests"] = r["responses"] + r["errors"]
    return r


def aggregate(rows, mode):
    parts = [x["hey"] for x in rows]
    resp = sum(p["responses"] for p in parts)
    req = sum(p["requests"] for p in parts)
    durs = [p["total_s"] or 0 for p in parts]
    dur = max(durs) if mode == "par" else sum(durs)
    codes = {}
    for p in parts:
        for c, n in p["codes"].items():
            codes[c] = codes.get(c, 0) + n

    def wavg(get):
        num = sum(get(p) * p["responses"] for p in parts if get(p) is not None and p["responses"])
        den = sum(p["responses"] for p in parts if get(p) is not None and p["responses"])
        return num / den if den else None

    ms = lambda v: None if v is None else v * 1000.0
    ok2xx = sum(n for c, n in codes.items() if c.startswith("2"))
    fast = [p["fastest_s"] for p in parts if p["fastest_s"] is not None]
    slow = [p["slowest_s"] for p in parts if p["slowest_s"] is not None]
    return {
        "requests": req, "responses": resp, "errors": sum(p["errors"] for p in parts),
        "clients": sum(int(x["clients"]) for x in rows) if mode == "par" else max(int(x["clients"]) for x in rows),
        "duration_s": dur, "rps": (resp / dur) if dur else None,
        "success_pct": (100.0 * ok2xx / req) if req else 0.0, "codes": codes,
        "n5xx": sum(n for c, n in codes.items() if c.startswith("5")),
        "min_ms": ms(min(fast)) if fast else None, "avg_ms": ms(wavg(lambda p: p["average_s"])),
        **{"p%s_ms" % k: ms(wavg(lambda p, k=k: p["pct"].get(k))) for k in ("50", "90", "95", "99")},
        "max_ms": ms(max(slow)) if slow else None,
    }


def f1(v, nd=1):
    return "-" if v is None else ("%.*f" % (nd, v))


def codes_str(c):
    return ", ".join("%s: %d" % (k, v) for k, v in sorted(c.items())) or "-"


def main():
    d = sys.argv[1]
    rows = []
    with open(os.path.join(d, "manifest.csv"), encoding="utf-8") as f:
        for x in csv.DictReader(f):
            x["hey"] = parse_hey(os.path.join(d, "raw", x["file"]))
            rows.append(x)
    order, by = [], {}
    for x in rows:
        if x["scenario"] not in by:
            order.append(x["scenario"])
            by[x["scenario"]] = []
        by[x["scenario"]].append(x)

    summary = []
    for s in order:
        agg = aggregate(by[s], by[s][0]["mode"])
        agg["scenario"] = s
        agg["k1"] = agg["success_pct"] >= 99.0
        agg["k2"] = agg["n5xx"] == 0
        agg["k3"] = agg["p95_ms"] is not None and agg["p95_ms"] <= 2000.0
        summary.append(agg)

    extra = {}
    xp = os.path.join(d, "extra.json")
    if os.path.exists(xp):
        extra = json.load(open(xp, encoding="utf-8"))
    k4 = None
    s1 = next((x for x in summary if x["scenario"] == "S1"), None)
    s6 = next((x for x in summary if x["scenario"] == "S6"), None)
    if s1 and s6:
        k4 = (extra.get("readyz_after") == 200 and s6["p50_ms"] is not None and s1["p50_ms"] is not None
              and s6["p50_ms"] <= 2 * s1["p50_ms"] + 1.0)

    hdr = ["Сцен.", "Запр.", "Клиентов", "Длит.,с", "RPS", "Успех,%", "min", "avg", "P50", "P90", "P95", "P99", "max", "Коды", "Ошибки", "Итог"]
    lines = []
    for a in summary:
        ok = a["k1"] and a["k2"] and a["k3"]
        lines.append([a["scenario"], str(a["requests"]), str(a["clients"]), f1(a["duration_s"]), f1(a["rps"], 0),
                      f1(a["success_pct"], 2), f1(a["min_ms"]), f1(a["avg_ms"]), f1(a["p50_ms"]), f1(a["p90_ms"]),
                      f1(a["p95_ms"]), f1(a["p99_ms"]), f1(a["max_ms"]), codes_str(a["codes"]), str(a["errors"]),
                      "PASS" if ok else "FAIL"])
    print_table("Сводные результаты (время — мс)", hdr, lines)

    for s in order:
        if len(by[s]) > 1 and s not in ("S1", "S6"):
            dl = []
            for x in by[s]:
                a = aggregate([x], "par")
                dl.append([x["label"], x["clients"], str(a["responses"]), f1(a["rps"], 1), f1(a["p50_ms"]),
                           f1(a["p95_ms"]), f1(a["p99_ms"]), f1(a["max_ms"]), codes_str(a["codes"]), str(a["errors"])])
            print_table("%s по эндпоинтам / прогонам (время — мс)" % s,
                        ["Эндпоинт / прогон", "Клиентов", "Ответов", "RPS", "P50", "P95", "P99", "max", "Коды", "Ошибки"], dl)

    if extra.get("uploads"):
        u = extra["uploads"]
        print("S5 загрузки: %d запросов POST /datasets, коды %s, время ответа %s–%s мс; обработано до ready: %d/%d, макс. время обработки %s с"
              % (u["count"], u["codes"], f1(u.get("min_ms")), f1(u.get("max_ms")), u.get("ready", 0), u["count"], f1(u.get("max_processing_s"), 2)))
    if extra.get("stats_peak"):
        print("\nПиковое потребление ресурсов (docker stats, сценарии S4/S5):")
        for name, v in sorted(extra["stats_peak"].items()):
            print("  %-34s CPU %6.1f %%   память %8.1f МиБ" % (name, v["cpu"], v["mem_mib"]))
    if k4 is not None:
        print("\nK4 (после нагрузки): /readyz %s; S6 P50/P95 = %s/%s мс против S1 %s/%s мс -> %s" % (
            extra.get("readyz_after"), f1(s6["p50_ms"]), f1(s6["p95_ms"]), f1(s1["p50_ms"]), f1(s1["p95_ms"]),
            "PASS" if k4 else "FAIL"))

    with open(os.path.join(d, "summary.csv"), "w", newline="", encoding="utf-8") as f:
        w = csv.writer(f)
        w.writerow(["scenario", "requests", "clients", "duration_s", "rps", "success_pct", "min_ms", "avg_ms",
                    "p50_ms", "p90_ms", "p95_ms", "p99_ms", "max_ms", "codes", "errors", "result"])
        for ln, a in zip(lines, summary):
            w.writerow([a["scenario"], a["requests"], a["clients"], a["duration_s"], a["rps"], a["success_pct"],
                        a["min_ms"], a["avg_ms"], a["p50_ms"], a["p90_ms"], a["p95_ms"], a["p99_ms"], a["max_ms"],
                        codes_str(a["codes"]), a["errors"], ln[-1]])
    with open(os.path.join(d, "detail.csv"), "w", newline="", encoding="utf-8") as f:
        w = csv.writer(f)
        w.writerow(["scenario", "label", "clients", "requests", "responses", "errors", "duration_s", "rps",
                    "min_ms", "avg_ms", "p50_ms", "p90_ms", "p95_ms", "p99_ms", "max_ms", "codes", "file"])
        for x in rows:
            a = aggregate([x], "par")
            w.writerow([x["scenario"], x["label"], x["clients"], a["requests"], a["responses"], a["errors"],
                        a["duration_s"], a["rps"], a["min_ms"], a["avg_ms"], a["p50_ms"], a["p90_ms"], a["p95_ms"],
                        a["p99_ms"], a["max_ms"], codes_str(a["codes"]), x["file"]])
    with open(os.path.join(d, "summary.json"), "w", encoding="utf-8") as f:
        json.dump({"scenarios": summary, "k4": k4, "extra": extra}, f, ensure_ascii=False, indent=2)

    failed = [a["scenario"] for a in summary if not (a["k1"] and a["k2"] and a["k3"])]
    if k4 is False:
        failed.append("K4")
    print("\nКритерии K1 (успех >= 99 %%), K2 (5xx = 0), K3 (P95 <= 2000 мс)%s: %s" % (
        ", K4" if k4 is not None else "", "все выполнены" if not failed else "НЕ выполнены: " + ", ".join(failed)))
    print("Файлы: %s/{summary.csv,summary.json,detail.csv,raw/}" % d)
    return 1 if failed else 0


def print_table(title, hdr, lines):
    widths = [max(len(str(r[i])) for r in [hdr] + lines) for i in range(len(hdr))]
    print("\n" + title)
    print(" | ".join(h.ljust(w) for h, w in zip(hdr, widths)))
    print("-+-".join("-" * w for w in widths))
    for r in lines:
        print(" | ".join(str(c).ljust(w) for c, w in zip(r, widths)))


if __name__ == "__main__":
    sys.exit(main())