infureal/gis
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(acceptance): reproducible Stage 8 acceptance package

Browse Source 
Add test/acceptance/ so the customer's reviewers can re-run what the
Stage 8 documents claim on an isolated stand built from this branch:

- compose.acceptance.yml + stand.sh: compose project gis-acceptance
  (app 18080, postgres 15432, minio 19000/19001, rabbitmq mgmt 15672),
  POSTGIS_PLATFORM / MINIO_IMAGE knobs for arm64 and amd64 hosts.
- functional.sh: the 89 test-stand checks of Приложение 7 section 5
  (F01-F38, N01-N38, L02-L05, L07-L15), own prefixed test data,
  PostGIS/MinIO confirmation via the stand containers, full cleanup.
- lp_gen.py / lp_run.sh / lp_verify.py: synthetic ЛП inputs, a run from
  stage 0, and an independent numpy re-implementation of every node that
  compares all 15 outputs pixel by pixel; read-only (GET) mode works
  against any stand, in row/column windows for 22219x14977 rasters.
- load.sh + seed.sh: scenarios S1-S6 of Приложение 3 with hey 0.1.5
  running inside the stand network; CSV/JSON summaries, criteria K1-K4.
- scan.sh: govulncheck (source and binary), gosec, trivy fs/image,
  golangci-lint; fails on any High/Critical.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bakhtiyar Issakhmetov
2026-10-15 22:37:28 +05:00
parent c6229c46f7
commit ca87d6961b
22 changed files with 3136 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
test/acceptance/lib/load_analyze.py Normal file
View File

@@ -0,0 +1,200 @@
#!/usr/bin/env python3
"""Aggregates the hey outputs of load.sh into the tables of Приложение 3.
load_analyze.py RESULTS_DIR
RESULTS_DIR/manifest.csv rows: scenario,mode,label,clients,file
mode par = the files of the scenario ran at the same time (duration = max)
seq = they ran one after another (duration = sum)
Each file is a default (summary) hey output. Per scenario the program reports
requests, clients, duration, RPS, success %, status codes, transport errors,
min/avg/P50/P90/P95/P99/max (ms). As in the protocol, percentiles of a
scenario made of several hey processes/runs are averages weighted by the
number of responses; min/max are exact.
Criteria (Приложение 3, табл. 8): K1 success >= 99 %, K2 no 5xx, K3 P95 <=
2000 ms; K4 (S6 vs S1): /readyz 200 and S6 P50 <= 2 x S1 P50 + 1 ms.
Writes summary.csv, summary.json, detail.csv into RESULTS_DIR and prints the
tables. Exit code 1 when a criterion fails.
"""
import csv
import json
import os
import re
import sys
PCTS = ["10", "25", "50", "75", "90", "95", "99"]
def parse_hey(path):
txt = open(path, encoding="utf-8", errors="replace").read()
def num(label):
m = re.search(r"^\s*%s:\s+([0-9.]+)" % re.escape(label), txt, re.M)
return float(m.group(1)) if m else None
r = {"total_s": num("Total"), "slowest_s": num("Slowest"), "fastest_s": num("Fastest"),
"average_s": num("Average"), "rps": num("Requests/sec"), "codes": {}, "errors": 0, "pct": {}}
for p, v in re.findall(r"^\s*(\d+)%+ in ([0-9.]+) secs", txt, re.M):
if p != "0": # hey 0.1.5 prints "%%" and a bogus "0%% in 0.0000" row for tiny samples
r["pct"][p] = float(v)
sec = txt.split("Status code distribution:", 1)
if len(sec) == 2:
for code, n in re.findall(r"^\s*\[(\d+)\]\s+(\d+) responses", sec[1].split("Error distribution:")[0], re.M):
r["codes"][code] = r["codes"].get(code, 0) + int(n)
sec = txt.split("Error distribution:", 1)
if len(sec) == 2:
r["errors"] = sum(int(n) for n in re.findall(r"^\s*\[(\d+)\]\s", sec[1], re.M))
r["responses"] = sum(r["codes"].values())
r["requests"] = r["responses"] + r["errors"]
return r
def aggregate(rows, mode):
parts = [x["hey"] for x in rows]
resp = sum(p["responses"] for p in parts)
req = sum(p["requests"] for p in parts)
durs = [p["total_s"] or 0 for p in parts]
dur = max(durs) if mode == "par" else sum(durs)
codes = {}
for p in parts:
for c, n in p["codes"].items():
codes[c] = codes.get(c, 0) + n
def wavg(get):
num = sum(get(p) * p["responses"] for p in parts if get(p) is not None and p["responses"])
den = sum(p["responses"] for p in parts if get(p) is not None and p["responses"])
return num / den if den else None
ms = lambda v: None if v is None else v * 1000.0
ok2xx = sum(n for c, n in codes.items() if c.startswith("2"))
fast = [p["fastest_s"] for p in parts if p["fastest_s"] is not None]
slow = [p["slowest_s"] for p in parts if p["slowest_s"] is not None]
return {
"requests": req, "responses": resp, "errors": sum(p["errors"] for p in parts),
"clients": sum(int(x["clients"]) for x in rows) if mode == "par" else max(int(x["clients"]) for x in rows),
"duration_s": dur, "rps": (resp / dur) if dur else None,
"success_pct": (100.0 * ok2xx / req) if req else 0.0, "codes": codes,
"n5xx": sum(n for c, n in codes.items() if c.startswith("5")),
"min_ms": ms(min(fast)) if fast else None, "avg_ms": ms(wavg(lambda p: p["average_s"])),
**{"p%s_ms" % k: ms(wavg(lambda p, k=k: p["pct"].get(k))) for k in ("50", "90", "95", "99")},
"max_ms": ms(max(slow)) if slow else None,
}
def f1(v, nd=1):
return "-" if v is None else ("%.*f" % (nd, v))
def codes_str(c):
return ", ".join("%s: %d" % (k, v) for k, v in sorted(c.items())) or "-"
def main():
d = sys.argv[1]
rows = []
with open(os.path.join(d, "manifest.csv"), encoding="utf-8") as f:
for x in csv.DictReader(f):
x["hey"] = parse_hey(os.path.join(d, "raw", x["file"]))
rows.append(x)
order, by = [], {}
for x in rows:
if x["scenario"] not in by:
order.append(x["scenario"])
by[x["scenario"]] = []
by[x["scenario"]].append(x)
summary = []
for s in order:
agg = aggregate(by[s], by[s][0]["mode"])
agg["scenario"] = s
agg["k1"] = agg["success_pct"] >= 99.0
agg["k2"] = agg["n5xx"] == 0
agg["k3"] = agg["p95_ms"] is not None and agg["p95_ms"] <= 2000.0
summary.append(agg)
extra = {}
xp = os.path.join(d, "extra.json")
if os.path.exists(xp):
extra = json.load(open(xp, encoding="utf-8"))
k4 = None
s1 = next((x for x in summary if x["scenario"] == "S1"), None)
s6 = next((x for x in summary if x["scenario"] == "S6"), None)
if s1 and s6:
k4 = (extra.get("readyz_after") == 200 and s6["p50_ms"] is not None and s1["p50_ms"] is not None
and s6["p50_ms"] <= 2 * s1["p50_ms"] + 1.0)
hdr = ["Сцен.", "Запр.", "Клиентов", "Длит.,с", "RPS", "Успех,%", "min", "avg", "P50", "P90", "P95", "P99", "max", "Коды", "Ошибки", "Итог"]
lines = []
for a in summary:
ok = a["k1"] and a["k2"] and a["k3"]
lines.append([a["scenario"], str(a["requests"]), str(a["clients"]), f1(a["duration_s"]), f1(a["rps"], 0),
f1(a["success_pct"], 2), f1(a["min_ms"]), f1(a["avg_ms"]), f1(a["p50_ms"]), f1(a["p90_ms"]),
f1(a["p95_ms"]), f1(a["p99_ms"]), f1(a["max_ms"]), codes_str(a["codes"]), str(a["errors"]),
"PASS" if ok else "FAIL"])
print_table("Сводные результаты (время — мс)", hdr, lines)
for s in order:
if len(by[s]) > 1 and s not in ("S1", "S6"):
dl = []
for x in by[s]:
a = aggregate([x], "par")
dl.append([x["label"], x["clients"], str(a["responses"]), f1(a["rps"], 1), f1(a["p50_ms"]),
f1(a["p95_ms"]), f1(a["p99_ms"]), f1(a["max_ms"]), codes_str(a["codes"]), str(a["errors"])])
print_table("%s по эндпоинтам / прогонам (время — мс)" % s,
["Эндпоинт / прогон", "Клиентов", "Ответов", "RPS", "P50", "P95", "P99", "max", "Коды", "Ошибки"], dl)
if extra.get("uploads"):
u = extra["uploads"]
print("S5 загрузки: %d запросов POST /datasets, коды %s, время ответа %s–%s мс; обработано до ready: %d/%d, макс. время обработки %s с"
% (u["count"], u["codes"], f1(u.get("min_ms")), f1(u.get("max_ms")), u.get("ready", 0), u["count"], f1(u.get("max_processing_s"), 2)))
if extra.get("stats_peak"):
print("\nПиковое потребление ресурсов (docker stats, сценарии S4/S5):")
for name, v in sorted(extra["stats_peak"].items()):
print(" %-34s CPU %6.1f %% память %8.1f МиБ" % (name, v["cpu"], v["mem_mib"]))
if k4 is not None:
print("\nK4 (после нагрузки): /readyz %s; S6 P50/P95 = %s/%s мс против S1 %s/%s мс -> %s" % (
extra.get("readyz_after"), f1(s6["p50_ms"]), f1(s6["p95_ms"]), f1(s1["p50_ms"]), f1(s1["p95_ms"]),
"PASS" if k4 else "FAIL"))
with open(os.path.join(d, "summary.csv"), "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["scenario", "requests", "clients", "duration_s", "rps", "success_pct", "min_ms", "avg_ms",
"p50_ms", "p90_ms", "p95_ms", "p99_ms", "max_ms", "codes", "errors", "result"])
for ln, a in zip(lines, summary):
w.writerow([a["scenario"], a["requests"], a["clients"], a["duration_s"], a["rps"], a["success_pct"],
a["min_ms"], a["avg_ms"], a["p50_ms"], a["p90_ms"], a["p95_ms"], a["p99_ms"], a["max_ms"],
codes_str(a["codes"]), a["errors"], ln[-1]])
with open(os.path.join(d, "detail.csv"), "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["scenario", "label", "clients", "requests", "responses", "errors", "duration_s", "rps",
"min_ms", "avg_ms", "p50_ms", "p90_ms", "p95_ms", "p99_ms", "max_ms", "codes", "file"])
for x in rows:
a = aggregate([x], "par")
w.writerow([x["scenario"], x["label"], x["clients"], a["requests"], a["responses"], a["errors"],
a["duration_s"], a["rps"], a["min_ms"], a["avg_ms"], a["p50_ms"], a["p90_ms"], a["p95_ms"],
a["p99_ms"], a["max_ms"], codes_str(a["codes"]), x["file"]])
with open(os.path.join(d, "summary.json"), "w", encoding="utf-8") as f:
json.dump({"scenarios": summary, "k4": k4, "extra": extra}, f, ensure_ascii=False, indent=2)
failed = [a["scenario"] for a in summary if not (a["k1"] and a["k2"] and a["k3"])]
if k4 is False:
failed.append("K4")
print("\nКритерии K1 (успех >= 99 %%), K2 (5xx = 0), K3 (P95 <= 2000 мс)%s: %s" % (
", K4" if k4 is not None else "", "все выполнены" if not failed else "НЕ выполнены: " + ", ".join(failed)))
print("Файлы: %s/{summary.csv,summary.json,detail.csv,raw/}" % d)
return 1 if failed else 0
def print_table(title, hdr, lines):
widths = [max(len(str(r[i])) for r in [hdr] + lines) for i in range(len(hdr))]
print("\n" + title)
print(" | ".join(h.ljust(w) for h, w in zip(hdr, widths)))
print("-+-".join("-" * w for w in widths))
for r in lines:
print(" | ".join(str(c).ljust(w) for c, w in zip(r, widths)))
if __name__ == "__main__":
sys.exit(main())