Bakhtiyar Issakhmetov
ca87d6961b
Add test/acceptance/ so the customer's reviewers can re-run what the Stage 8 documents claim on an isolated stand built from this branch: - compose.acceptance.yml + stand.sh: compose project gis-acceptance (app 18080, postgres 15432, minio 19000/19001, rabbitmq mgmt 15672), POSTGIS_PLATFORM / MINIO_IMAGE knobs for arm64 and amd64 hosts. - functional.sh: the 89 test-stand checks of Приложение 7 section 5 (F01-F38, N01-N38, L02-L05, L07-L15), own prefixed test data, PostGIS/MinIO confirmation via the stand containers, full cleanup. - lp_gen.py / lp_run.sh / lp_verify.py: synthetic ЛП inputs, a run from stage 0, and an independent numpy re-implementation of every node that compares all 15 outputs pixel by pixel; read-only (GET) mode works against any stand, in row/column windows for 22219x14977 rasters. - load.sh + seed.sh: scenarios S1-S6 of Приложение 3 with hey 0.1.5 running inside the stand network; CSV/JSON summaries, criteria K1-K4. - scan.sh: govulncheck (source and binary), gosec, trivy fs/image, golangci-lint; fails on any High/Critical. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
102 lines
5.0 KiB
Bash
Executable File
102 lines
5.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# Seeds the acceptance stand with datasets for the load test (Приложение 3,
|
||
# п. 3.3), so the read endpoints work on a realistic catalogue.
|
||
#
|
||
# test/acceptance/seed.sh create (idempotent) category "acc-load":
|
||
# SEED_VECTOR (25) vector GeoJSON,
|
||
# SEED_KATO (25) vector_with_kato GeoJSON with
|
||
# KATO mapping (14 districts x 3 years each),
|
||
# SEED_RASTER (50) GeoTIFF rasters; the
|
||
# vector_with_kato datasets form the group
|
||
# "acc-load-population"
|
||
# test/acceptance/seed.sh clean delete everything seed.sh created
|
||
#
|
||
# Writes the ids load.sh needs to test/acceptance/results/seed.env.
|
||
# WRITES to the stand: never point it at production.
|
||
set -euo pipefail
|
||
|
||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||
BASE_URL="${BASE_URL:-http://localhost:18080}"
|
||
BASE_URL="${BASE_URL%/}"
|
||
PYTHON="${PYTHON:-python3}"
|
||
SEED_VECTOR="${SEED_VECTOR:-25}"
|
||
SEED_KATO="${SEED_KATO:-25}"
|
||
SEED_RASTER="${SEED_RASTER:-50}"
|
||
CAT_CODE=acc-load
|
||
GRP_CODE=acc-load-population
|
||
ENV_OUT="$HERE/results/seed.env"
|
||
mkdir -p "$HERE/results"
|
||
case "$BASE_URL" in
|
||
*dssgis.dwh.kz*) echo "refusing to write to $BASE_URL (production)" >&2; exit 2;;
|
||
esac
|
||
J=(-H 'Content-Type: application/json')
|
||
|
||
cat_id() { curl -fsS "$BASE_URL/categories" | jq -r --arg c "$CAT_CODE" '.[]|select(.code==$c)|.id' | head -n1; }
|
||
grp_id() { curl -fsS "$BASE_URL/dataset-groups?code=$GRP_CODE" | jq -r '.data[0].id // empty'; }
|
||
ids_in_category() { # all dataset ids of a category (paginated)
|
||
local cat="$1" page=1 out
|
||
while :; do
|
||
out="$(curl -fsS "$BASE_URL/datasets?category_id=$cat&page_size=100&page=$page")"
|
||
jq -r '.data[].id' <<< "$out"
|
||
[ "$page" -lt "$(jq -r .total_pages <<< "$out")" ] || break
|
||
page=$((page + 1))
|
||
done
|
||
}
|
||
|
||
if [ "${1:-}" = clean ]; then
|
||
CAT="$(cat_id || true)"; GRP="$(grp_id || true)"
|
||
if [ -n "$CAT" ]; then
|
||
n=0; for id in $(ids_in_category "$CAT"); do curl -sS -o /dev/null -X DELETE "$BASE_URL/datasets/$id"; n=$((n + 1)); done
|
||
echo "deleted $n datasets"
|
||
fi
|
||
[ -n "$GRP" ] && curl -sS -o /dev/null -w "group delete %{http_code}\n" -X DELETE "$BASE_URL/dataset-groups/$GRP"
|
||
[ -n "$CAT" ] && curl -sS -o /dev/null -w "category delete %{http_code}\n" -X DELETE "$BASE_URL/categories/$CAT"
|
||
rm -f "$ENV_OUT"
|
||
exit 0
|
||
fi
|
||
|
||
CAT="$(cat_id || true)"
|
||
if [ -z "$CAT" ]; then
|
||
CAT="$(curl -fsS -X POST "$BASE_URL/categories" "${J[@]}" -d "{\"code\":\"$CAT_CODE\",\"name\":\"Нагрузочное тестирование\",\"description\":\"Данные для test/acceptance/load.sh\"}" | jq -r .id)"
|
||
fi
|
||
GRP="$(grp_id || true)"
|
||
if [ -z "$GRP" ]; then
|
||
GRP="$(curl -fsS -X POST "$BASE_URL/dataset-groups" "${J[@]}" -d "{\"code\":\"$GRP_CODE\",\"name\":\"Население СКО (нагрузочный тест)\"}" | jq -r .id)"
|
||
fi
|
||
|
||
want=$((SEED_VECTOR + SEED_KATO + SEED_RASTER))
|
||
have="$(curl -fsS "$BASE_URL/datasets?category_id=$CAT&status=ready&page_size=1" | jq -r .total)"
|
||
if [ "$have" -lt "$want" ]; then
|
||
echo "seeding $BASE_URL: $SEED_VECTOR vector, $SEED_KATO vector_with_kato, $SEED_RASTER raster (category $CAT_CODE)"
|
||
TMP="$(mktemp -d "${TMPDIR:-/tmp}/gis-seed.XXXXXX")"; trap 'rm -rf "$TMP"' EXIT
|
||
"$PYTHON" "$HERE/lib/helpers.py" ndvi "$TMP/ndvi_seed.tif"
|
||
upload() { curl -fsS -X POST "$BASE_URL/datasets" "$@" | jq -r .id; }
|
||
for i in $(seq 1 "$SEED_VECTOR"); do
|
||
upload -F "file=@$HERE/data/plots_nko.geojson" -F file_type=vector -F "category_id=$CAT" -F "name=Участки $i" >/dev/null
|
||
done
|
||
kato_ids=""
|
||
for i in $(seq 1 "$SEED_KATO"); do
|
||
y=$((2000 + i))
|
||
kato_ids="$kato_ids $(upload -F "file=@$HERE/data/population_kato.geojson" -F file_type=vector_with_kato -F "category_id=$CAT" \
|
||
-F "name=Население $i" -F "dg_id=$GRP" -F "start_date=$y-01-01" -F "end_date=$y-12-31")"
|
||
done
|
||
for i in $(seq 1 "$SEED_RASTER"); do
|
||
upload -F "file=@$TMP/ndvi_seed.tif" -F file_type=raster -F "category_id=$CAT" -F "name=NDVI $i" >/dev/null
|
||
done
|
||
MAP='{"kato_column":"kato","year_columns":[{"column":"Y2022","start_date":"2022-01-01","end_date":"2022-12-31"},{"column":"Y2023","start_date":"2023-01-01","end_date":"2023-12-31"},{"column":"Y2024","start_date":"2024-01-01","end_date":"2024-12-31"}]}'
|
||
for id in $kato_ids; do
|
||
for _ in $(seq 1 120); do
|
||
[ "$(curl -fsS "$BASE_URL/datasets/$id" | jq -r .status)" = awaiting_mapping ] && break; sleep 1
|
||
done
|
||
curl -fsS -o /dev/null -X POST "$BASE_URL/datasets/$id/mapping" "${J[@]}" -d "$MAP"
|
||
done
|
||
for _ in $(seq 1 300); do
|
||
have="$(curl -fsS "$BASE_URL/datasets?category_id=$CAT&status=ready&page_size=1" | jq -r .total)"
|
||
[ "$have" -ge "$want" ] && break; sleep 1
|
||
done
|
||
fi
|
||
VWK="$(curl -fsS "$BASE_URL/datasets?category_id=$CAT&file_type=vector_with_kato&status=ready&page_size=1" | jq -r '.data[0].id')"
|
||
printf 'CAT_ID=%s\nGROUP_ID=%s\nVWK_ID=%s\n' "$CAT" "$GRP" "$VWK" > "$ENV_OUT"
|
||
echo "seed: category $CAT_CODE has $have ready datasets (wanted $want); ids in $ENV_OUT"
|
||
cat "$ENV_OUT"
|