test(acceptance): reproducible Stage 8 acceptance package
Add test/acceptance/ so the customer's reviewers can re-run what the Stage 8 documents claim on an isolated stand built from this branch: - compose.acceptance.yml + stand.sh: compose project gis-acceptance (app 18080, postgres 15432, minio 19000/19001, rabbitmq mgmt 15672), POSTGIS_PLATFORM / MINIO_IMAGE knobs for arm64 and amd64 hosts. - functional.sh: the 89 test-stand checks of Приложение 7 section 5 (F01-F38, N01-N38, L02-L05, L07-L15), own prefixed test data, PostGIS/MinIO confirmation via the stand containers, full cleanup. - lp_gen.py / lp_run.sh / lp_verify.py: synthetic ЛП inputs, a run from stage 0, and an independent numpy re-implementation of every node that compares all 15 outputs pixel by pixel; read-only (GET) mode works against any stand, in row/column windows for 22219x14977 rasters. - load.sh + seed.sh: scenarios S1-S6 of Приложение 3 with hey 0.1.5 running inside the stand network; CSV/JSON summaries, criteria K1-K4. - scan.sh: govulncheck (source and binary), gosec, trivy fs/image, golangci-lint; fails on any High/Critical. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
758
test/acceptance/functional.sh
Executable file
758
test/acceptance/functional.sh
Executable file
@@ -0,0 +1,758 @@
|
||||
#!/usr/bin/env bash
|
||||
# Functional acceptance test of the GIS service on a test stand — the 89 checks
|
||||
# of section 5 of «Протокол функционального тестирования» (Приложение 7):
|
||||
# F01–F38 positive scenarios (categories, datasets, groups, SLD, filters,
|
||||
# PostGIS / MinIO confirmation)
|
||||
# N01–N38 negative scenarios
|
||||
# L02–L05, L07–L15 Landscape Program (ЛП) on synthetic rasters, incl. the
|
||||
# pixel-by-pixel reference check (lp_verify.py)
|
||||
#
|
||||
# Usage:
|
||||
# test/acceptance/functional.sh # stand at http://localhost:18080
|
||||
# BASE_URL=http://host:port test/acceptance/functional.sh
|
||||
#
|
||||
# Environment:
|
||||
# BASE_URL API address (default http://localhost:18080)
|
||||
# LOCAL_CHECKS 1 = confirm in PostGIS/MinIO through the docker containers of
|
||||
# the compose project (F35 MinIO part, F36, F37, F38, L15
|
||||
# resource figures); 0 = skip those (reported as SKIP);
|
||||
# auto (default) = 1 when the gis-acceptance containers run
|
||||
# COMPOSE_PROJECT compose project of the stand (default gis-acceptance)
|
||||
# PYTHON python3 with numpy + rasterio (default python3)
|
||||
# KEEP_DATA 1 = do not delete the created test data at the end
|
||||
# WAIT_TIMEOUT seconds to wait for asynchronous processing (default 300)
|
||||
#
|
||||
# Output: one line per check "ID | описание | PASS/FAIL/SKIP | detail", then a
|
||||
# summary. Exit code 0 only when there is no FAIL. Every object the script
|
||||
# creates carries a unique run prefix and is deleted at the end (unless
|
||||
# KEEP_DATA=1); pre-existing objects are never modified or deleted. The script
|
||||
# writes the ЛП slots, so run it on an isolated test stand, never on production.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
DATA="$HERE/data"
|
||||
BASE_URL="${BASE_URL:-http://localhost:18080}"
|
||||
BASE_URL="${BASE_URL%/}"
|
||||
COMPOSE_PROJECT="${COMPOSE_PROJECT:-gis-acceptance}"
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
KEEP_DATA="${KEEP_DATA:-0}"
|
||||
WAIT_TIMEOUT="${WAIT_TIMEOUT:-300}"
|
||||
LOCAL_CHECKS="${LOCAL_CHECKS:-auto}"
|
||||
|
||||
for bin in curl jq "$PYTHON"; do
|
||||
command -v "$bin" >/dev/null 2>&1 || { echo "missing required tool: $bin" >&2; exit 2; }
|
||||
done
|
||||
"$PYTHON" -c 'import numpy, rasterio' 2>/dev/null || { echo "$PYTHON needs numpy and rasterio (pip install -r $HERE/requirements.txt)" >&2; exit 2; }
|
||||
|
||||
container_of() { # service -> container id of the compose project (empty if none)
|
||||
docker ps -q --filter "label=com.docker.compose.project=$COMPOSE_PROJECT" \
|
||||
--filter "label=com.docker.compose.service=$1" 2>/dev/null | head -n1
|
||||
}
|
||||
if [ "$LOCAL_CHECKS" = auto ]; then
|
||||
LOCAL_CHECKS=0
|
||||
if command -v docker >/dev/null 2>&1 && [ -n "$(container_of postgres)" ] && [ -n "$(container_of minio)" ]; then
|
||||
LOCAL_CHECKS=1
|
||||
fi
|
||||
fi
|
||||
PG_C=""; MINIO_C=""; WORKER_C=""
|
||||
if [ "$LOCAL_CHECKS" = 1 ]; then
|
||||
PG_C="$(container_of postgres)"; MINIO_C="$(container_of minio)"; WORKER_C="$(container_of worker)"
|
||||
fi
|
||||
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
P="acc$(date +%s)" # unique slug prefix of every object this run creates
|
||||
TMP="$(mktemp -d "${TMPDIR:-/tmp}/gis-func.XXXXXX")"
|
||||
RESULTS="$HERE/results/functional-$STAMP"
|
||||
mkdir -p "$RESULTS"
|
||||
BODY="$TMP/body"
|
||||
JSON=(-H 'Content-Type: application/json')
|
||||
|
||||
# ------------------------------------------------------------------ reporting
|
||||
PASSN=0; FAILN=0; SKIPN=0
|
||||
CUR_ID=""; CUR_DESC=""; CUR_OK=1; CUR_MSG=""
|
||||
begin() { CUR_ID="$1"; CUR_DESC="$2"; CUR_OK=1; CUR_MSG=""; }
|
||||
bad() { CUR_OK=0; CUR_MSG="${CUR_MSG:+$CUR_MSG; }$1"; }
|
||||
line() { printf '%-4s | %s | %s | %s\n' "$1" "$2" "$3" "$4" | tee -a "$RESULTS/functional.txt"; }
|
||||
end() { # detail
|
||||
if [ "$CUR_OK" = 1 ]; then PASSN=$((PASSN + 1)); line "$CUR_ID" "$CUR_DESC" PASS "$1"
|
||||
else FAILN=$((FAILN + 1)); line "$CUR_ID" "$CUR_DESC" FAIL "$CUR_MSG"; fi
|
||||
}
|
||||
skip() { SKIPN=$((SKIPN + 1)); line "$CUR_ID" "$CUR_DESC" SKIP "$1"; }
|
||||
snippet() { head -c 220 "$BODY" 2>/dev/null | tr '\n\r' ' '; }
|
||||
|
||||
# ----------------------------------------------------------------------- HTTP
|
||||
CODE=000; TIME=0
|
||||
http() { # METHOD PATH [curl args...] -> $CODE, $TIME (s), body in $BODY
|
||||
local m="$1" p="$2" out; shift 2
|
||||
out="$(curl -sS -o "$BODY" -w '%{http_code} %{time_total}' -X "$m" "$BASE_URL$p" "$@" 2>"$TMP/curl.err")" || out="000 0"
|
||||
CODE="${out%% *}"; TIME="${out#* }"
|
||||
}
|
||||
jb() { jq -r "$1" "$BODY" 2>/dev/null || true; }
|
||||
jbc() { jq -c "${1:-.}" "$BODY" 2>/dev/null || snippet; }
|
||||
need_code() { [ "$CODE" = "$1" ] || bad "HTTP $CODE (ожидалось $1): $(snippet)"; }
|
||||
need_eq() { [ "$1" = "$2" ] || bad "$3: «$1» ≠ «$2»"; }
|
||||
need_has() { case "$1" in *"$2"*) ;; *) bad "$3: нет «$2» в «${1:0:200}»";; esac; }
|
||||
need_true() { [ "$1" = true ] || bad "$2"; }
|
||||
|
||||
sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi; }
|
||||
|
||||
# dataset status polling: wait_ds ID STATUS... -> $WS (final status)
|
||||
WS=""
|
||||
wait_ds() {
|
||||
local id="$1"; shift
|
||||
local deadline=$(( $(date +%s) + WAIT_TIMEOUT ))
|
||||
while :; do
|
||||
WS="$(curl -sS "$BASE_URL/datasets/$id" | jq -r '.status' 2>/dev/null || echo "?")"
|
||||
for s in "$@"; do [ "$WS" = "$s" ] && return 0; done
|
||||
[ "$WS" = failed ] && return 0
|
||||
[ "$(date +%s)" -ge "$deadline" ] && return 0
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
# run polling: wait_run ID -> $RS final status; run JSON in $TMP/run-ID.json
|
||||
RS=""
|
||||
wait_run() {
|
||||
local id="$1" deadline=$(( $(date +%s) + WAIT_TIMEOUT ))
|
||||
while :; do
|
||||
curl -sS "$BASE_URL/landscape/runs/$id" -o "$TMP/run-$id.json" || true
|
||||
RS="$(jq -r '.status' "$TMP/run-$id.json" 2>/dev/null || echo "?")"
|
||||
case "$RS" in succeeded|failed) return 0;; esac
|
||||
[ "$(date +%s)" -ge "$deadline" ] && return 0
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
node_statuses() { jq -r '[.nodes[] | "\(.node_code):\(.status)"] | sort | join(" ")' "$TMP/run-$1.json" 2>/dev/null || true; }
|
||||
wait_outputs() { # RUN: wait until every output dataset of the run is ready (COG done)
|
||||
local id; for id in $(run_outputs "$1" | jq -r '.[].dataset_id'); do wait_ds "$id" ready; done
|
||||
}
|
||||
run_outputs() { jq -c '[.nodes[] | .outputs[]?] | sort_by(.group)' "$TMP/run-$1.json" 2>/dev/null || echo '[]'; }
|
||||
|
||||
# ------------------------------------------------------------- cleanup ledger
|
||||
DATASETS=""; GRPS=""; CATS=""; RUNS=""
|
||||
track_ds() { [ -n "$1" ] && [ "$1" != null ] && DATASETS="$DATASETS $1"; return 0; }
|
||||
cleanup() {
|
||||
local rc=$?
|
||||
if [ "$KEEP_DATA" = 1 ]; then
|
||||
echo "KEEP_DATA=1: test data kept (prefix $P)"
|
||||
else
|
||||
local ids="$DATASETS" r d
|
||||
for r in $RUNS; do # outputs computed by our runs (never reused foreign ones)
|
||||
[ -f "$TMP/run-$r.json" ] || continue
|
||||
ids="$ids $(jq -r '[.nodes[] | select(.status=="succeeded") | .outputs[]?.dataset_id] | join(" ")' "$TMP/run-$r.json" 2>/dev/null || true)"
|
||||
done
|
||||
for d in $ids; do curl -sS -o /dev/null -X DELETE "$BASE_URL/datasets/$d" || true; done
|
||||
for d in $GRPS; do curl -sS -o /dev/null -X DELETE "$BASE_URL/dataset-groups/$d" || true; done
|
||||
for d in $CATS; do curl -sS -o /dev/null -X DELETE "$BASE_URL/categories/$d" || true; done
|
||||
echo "cleanup: removed test datasets, groups and categories with prefix $P (ЛП run records are kept: the API has no run deletion)"
|
||||
fi
|
||||
rm -rf "$TMP"
|
||||
exit "$rc"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "functional acceptance test: $BASE_URL prefix=$P local_checks=$LOCAL_CHECKS log=$RESULTS"
|
||||
: > "$RESULTS/functional.txt"
|
||||
|
||||
# ====================================================================== 5.2
|
||||
begin F01 "Создание корневой категории POST /categories"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-ecology\",\"name\":\"Экология\",\"description\":\"Приёмочный тест\"}"
|
||||
need_code 201; CAT_ECO="$(jb .id)"; need_eq "$(jb .parent_id)" null "parent_id"
|
||||
[ -n "$CAT_ECO" ] && [ "$CAT_ECO" != null ] && CATS="$CAT_ECO $CATS"
|
||||
end "201; $P-ecology «Экология», parent_id null"
|
||||
|
||||
begin F02 "Создание дочерних категорий (parent_id)"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-ecology-soil\",\"name\":\"Почвы\",\"parent_id\":\"$CAT_ECO\"}"
|
||||
need_code 201; CAT_SOIL="$(jb .id)"; need_eq "$(jb .parent_id)" "$CAT_ECO" "parent_id soil"
|
||||
[ -n "$CAT_SOIL" ] && [ "$CAT_SOIL" != null ] && CATS="$CAT_SOIL $CATS"
|
||||
c1="$CODE"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-ecology-soil-humus\",\"name\":\"Гумус\",\"parent_id\":\"$CAT_SOIL\"}"
|
||||
need_code 201; CAT_HUM="$(jb .id)"; need_eq "$(jb .parent_id)" "$CAT_SOIL" "parent_id humus"
|
||||
[ -n "$CAT_HUM" ] && [ "$CAT_HUM" != null ] && CATS="$CAT_HUM $CATS"
|
||||
end "$c1 / $CODE: трёхуровневая иерархия ecology → soil → humus"
|
||||
|
||||
begin F03 "Список категорий с фильтром parent_id"
|
||||
http GET "/categories?parent_id=$CAT_ECO"
|
||||
need_code 200; need_eq "$(jb 'length')" 1 "число дочерних"; need_eq "$(jb '.[0].id')" "$CAT_SOIL" "дочерняя категория"
|
||||
end "200; возвращена дочерняя категория $P-ecology-soil"
|
||||
|
||||
begin F04 "Получение категории по id"
|
||||
http GET "/categories/$CAT_SOIL"
|
||||
need_code 200
|
||||
need_eq "$(jb '[has("id"),has("parent_id"),has("code"),has("name"),has("description")]|all')" true "состав полей"
|
||||
SOIL_UPDATED="$(jb .updated_at)"
|
||||
end "200; поля id, parent_id, code, name, description"
|
||||
|
||||
begin F05 "Обновление категории (PUT)"
|
||||
sleep 1
|
||||
http PUT "/categories/$CAT_SOIL" "${JSON[@]}" -d "{\"code\":\"$P-ecology-soil\",\"name\":\"Почвы и грунты\",\"description\":\"Обновлено\",\"parent_id\":\"$CAT_ECO\"}"
|
||||
need_code 200; need_eq "$(jb .name)" "Почвы и грунты" name; need_eq "$(jb .description)" "Обновлено" description
|
||||
[ "$(jb .updated_at)" != "$SOIL_UPDATED" ] || bad "updated_at не изменился"
|
||||
end "200; name «Почвы и грунты», description «Обновлено», updated_at обновлён"
|
||||
|
||||
begin F06 "Цикл в иерархии (корень → внук)"
|
||||
http PUT "/categories/$CAT_ECO" "${JSON[@]}" -d "{\"code\":\"$P-ecology\",\"name\":\"Экология\",\"parent_id\":\"$CAT_HUM\"}"
|
||||
need_code 422; need_has "$(jb .error)" "own ancestor" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
|
||||
begin F07 "Категория-родитель самой себя"
|
||||
http PUT "/categories/$CAT_ECO" "${JSON[@]}" -d "{\"code\":\"$P-ecology\",\"name\":\"Экология\",\"parent_id\":\"$CAT_ECO\"}"
|
||||
need_code 422; need_has "$(jb .error)" "own ancestor" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
|
||||
begin F08 "Дубликат кода категории"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-ecology\",\"name\":\"Дубликат\"}"
|
||||
need_code 409
|
||||
[ "$CODE" = 201 ] && CATS="$(jb .id) $CATS"
|
||||
end "409 «$(jb .error)»"
|
||||
|
||||
begin F09 "Удаление категории"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-tmp\",\"name\":\"Временная\"}"
|
||||
need_code 201; tmpc="$(jb .id)"; c1="$CODE"
|
||||
http DELETE "/categories/$tmpc"; need_code 204; c2="$CODE"
|
||||
http GET "/categories/$tmpc"; need_code 404
|
||||
end "$c1 (создана) → $c2 (удалена) → повторный GET $CODE"
|
||||
|
||||
# ====================================================================== 5.3
|
||||
begin F10 "Создание группы наборов POST /dataset-groups"
|
||||
http POST /dataset-groups "${JSON[@]}" -d "{\"code\":\"$P-population-nko\",\"name\":\"Численность населения СКО\",\"description\":\"Приёмочный тест\"}"
|
||||
need_code 201; GRP="$(jb .id)"
|
||||
[ -n "$GRP" ] && [ "$GRP" != null ] && GRPS="$GRP $GRPS"
|
||||
end "201; группа $P-population-nko «Численность населения СКО»"
|
||||
|
||||
begin F11 "Загрузка набора vector (GeoJSON)"
|
||||
http POST /datasets -F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F "category_id=$CAT_ECO" \
|
||||
-F "name=Участки СКО" -F "description=Полигоны участков" -F "unit=ha"
|
||||
need_code 201; DS_PLOTS="$(jb .id)"; track_ds "$DS_PLOTS"; s0="$(jb .status)"; key="$(jb .storage_key)"
|
||||
need_has "$key" "/plots_nko.geojson" storage_key
|
||||
wait_ds "$DS_PLOTS" ready; need_eq "$WS" ready status
|
||||
end "201, status $s0 → $WS; storage_key …/plots_nko.geojson"
|
||||
|
||||
begin F12 "Выгрузка vector в GeoJSON"
|
||||
http GET "/datasets/$DS_PLOTS.geojson"
|
||||
need_code 200; need_eq "$(jb .type)" FeatureCollection type; need_eq "$(jb '.features|length')" 4 "число объектов"
|
||||
need_eq "$(jb '[.features[].geometry.type]|unique|join(",")')" Polygon "тип геометрии"
|
||||
ext="$(jb '[.. | arrays | select(length==2 and (.[0]|type)=="number")] as $p | "\([$p[][0]]|min),\([$p[][1]]|min) – \([$p[][0]]|max),\([$p[][1]]|max)"')"
|
||||
need_eq "$(jb '[.. | arrays | select(length==2 and (.[0]|type)=="number")] as $p | ([$p[][0]]|min)==69 and ([$p[][1]]|min)==53.6 and ([$p[][0]]|max)==71.7 and ([$p[][1]]|max)==55')" true "охват (69.0, 53.6) – (71.7, 55.0): $ext"
|
||||
end "200; FeatureCollection, 4 объекта Polygon; охват ($ext)"
|
||||
|
||||
begin F13 "Загрузка vector_with_kato (GeoJSON, реальные коды КАТО)"
|
||||
http POST /datasets -F "file=@$DATA/population_kato.geojson" -F file_type=vector_with_kato -F "category_id=$CAT_ECO" \
|
||||
-F "name=Численность населения" -F unit=чел -F "dg_id=$GRP" -F start_date=2022-01-01 -F end_date=2024-12-31
|
||||
need_code 201; DS_POP="$(jb .id)"; track_ds "$DS_POP"; s0="$(jb .status)"; need_eq "$(jb .dg_id)" "$GRP" dg_id
|
||||
wait_ds "$DS_POP" awaiting_mapping; need_eq "$WS" awaiting_mapping status
|
||||
end "201, status $s0 → $WS, dg_id = группа $P-population-nko"
|
||||
|
||||
begin F14 "Обнаружение колонок"
|
||||
http GET "/datasets/$DS_POP"
|
||||
need_code 200; need_eq "$(jb .status)" awaiting_mapping status
|
||||
cols="$(jb '[.attribute_columns[].name]|join(",")')"
|
||||
need_eq "$cols" "kato,district,Y2022,Y2023,Y2024" колонки
|
||||
need_eq "$(jb '[.attribute_columns[]|(.samples|length)>0]|all')" true "примеры значений"
|
||||
end "200; колонки $cols с примерами значений"
|
||||
|
||||
MAP="$TMP/map.json"
|
||||
mapping() { http POST "/datasets/$DS_POP/mapping" "${JSON[@]}" -d "$1"; }
|
||||
begin N01 "Маппинг: несуществующая колонка года"
|
||||
mapping '{"kato_column":"kato","year_columns":[{"column":"Y1999","start_date":"2022-01-01"}]}'
|
||||
need_code 422; need_has "$(jb .error)" 'year column "Y1999" is not among the detected columns' сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N02 "Маппинг: несуществующая колонка КАТО"
|
||||
mapping '{"kato_column":"KATO_X","year_columns":[{"column":"Y2022","start_date":"2022-01-01"}]}'
|
||||
need_code 422; need_has "$(jb .error)" 'kato_column "KATO_X" is not among the detected columns' сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N03 "Маппинг: end_date раньше start_date"
|
||||
mapping '{"kato_column":"kato","year_columns":[{"column":"Y2022","start_date":"2022-12-31","end_date":"2022-01-01"}]}'
|
||||
need_code 422; need_has "$(jb .error)" 'is before start_date' сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N04 "Маппинг: некорректная дата 2022-13-45"
|
||||
mapping '{"kato_column":"kato","year_columns":[{"column":"Y2022","start_date":"2022-13-45"}]}'
|
||||
need_code 422; need_eq "$(jb .errors.StartDate)" "is invalid" errors.StartDate
|
||||
end "422 $(jbc)"
|
||||
begin N05 "Маппинг: пустой список year_columns"
|
||||
mapping '{"kato_column":"kato","year_columns":[]}'
|
||||
need_code 422; need_has "$(jb .errors.YearColumns)" "at least 1" errors.YearColumns
|
||||
end "422 $(jbc)"
|
||||
|
||||
begin F15 "Сохранение маппинга КАТО и годовых колонок с периодами"
|
||||
mapping '{"kato_column":"kato","year_columns":[{"column":"Y2022","start_date":"2022-01-01","end_date":"2022-12-31"},{"column":"Y2023","start_date":"2023-01-01","end_date":"2023-12-31"},{"column":"Y2024","start_date":"2024-01-01","end_date":"2024-12-31"}]}'
|
||||
need_code 200; s0="$(jb .status)"; need_eq "$(jb '.year_columns|length')" 3 year_columns; need_eq "$(jb .kato_column)" kato kato_column
|
||||
wait_ds "$DS_POP" ready; need_eq "$WS" ready status
|
||||
end "200, status $s0 → $WS; 3 колонки Y2022–Y2024 с периодами"
|
||||
|
||||
fixval() { jq -r --arg k "$1" --arg y "$2" '.features[]|select(.properties.kato==$k)|.properties[$y]' "$DATA/population_kato.geojson"; }
|
||||
begin F16 "Наблюдения (long format)"
|
||||
http GET "/datasets/$DS_POP/observations?page_size=100"
|
||||
need_code 200; need_eq "$(jb .total)" 42 total
|
||||
exp="$(fixval 591000000 Y2022)"
|
||||
got="$(jb '.data[]|select(.kato_code=="591000000" and .start_date=="2022-01-01")|.value')"
|
||||
need_eq "$got" "$exp" "591000000 / 2022"
|
||||
end "200; total $(jb .total) (14 районов × 3 года), 591000000 / 2022 → $got"
|
||||
|
||||
begin F17 "Фильтр наблюдений по kato_code"
|
||||
http GET "/datasets/$DS_POP/observations?kato_code=595000000"
|
||||
need_code 200; need_eq "$(jb .total)" 3 total
|
||||
got="$(jb '[.data|sort_by(.start_date)[]|.value]|join(" / ")')"
|
||||
exp="$(fixval 595000000 Y2022) / $(fixval 595000000 Y2023) / $(fixval 595000000 Y2024)"
|
||||
need_eq "$got" "$exp" "значения 595000000"
|
||||
end "200; для 595000000 — $(jb .total) наблюдения: $got"
|
||||
|
||||
begin F18 "GeoJSON vector_with_kato: соединение с districts"
|
||||
http GET "/datasets/$DS_POP.geojson"
|
||||
need_code 200; need_eq "$(jb '.features|length')" 14 "число объектов"
|
||||
need_eq "$(jb '[.features[].geometry.type]|unique|join(",")')" MultiPolygon "тип геометрии"
|
||||
need_eq "$(jb '.features[0].properties|keys|join(",")')" "data,kato,name,periods" "свойства"
|
||||
end "200; 14 объектов MultiPolygon, свойства data, kato, name, periods"
|
||||
|
||||
begin F19 "Загрузка vector_with_kato (GeoPackage)"
|
||||
http POST /datasets -F "file=@$DATA/livestock_kato.gpkg" -F file_type=vector_with_kato -F "category_id=$CAT_ECO" \
|
||||
-F "name=Поголовье КРС" -F unit=голов
|
||||
need_code 201; DS_LIVE="$(jb .id)"; track_ds "$DS_LIVE"; c1="$CODE"
|
||||
wait_ds "$DS_LIVE" awaiting_mapping; need_eq "$WS" awaiting_mapping status
|
||||
http GET "/datasets/$DS_LIVE"; cols="$(jb '[.attribute_columns[].name]|join(",")')"
|
||||
need_eq "$cols" "fid,KATO_CODE,district,H_2022,H_2023,H_2024" колонки
|
||||
http POST "/datasets/$DS_LIVE/mapping" "${JSON[@]}" -d '{"kato_column":"KATO_CODE","year_columns":[{"column":"H_2022","start_date":"2022-01-01","end_date":"2022-12-31"},{"column":"H_2023","start_date":"2023-01-01","end_date":"2023-12-31"},{"column":"H_2024","start_date":"2024-01-01","end_date":"2024-12-31"}]}'
|
||||
need_code 200; c2="$CODE"
|
||||
wait_ds "$DS_LIVE" ready; need_eq "$WS" ready status
|
||||
end "$c1 → awaiting_mapping; колонки $cols; маппинг $c2, статус $WS"
|
||||
|
||||
NDVI="$TMP/ndvi_nko.tif"
|
||||
"$PYTHON" "$HERE/lib/helpers.py" ndvi "$NDVI"
|
||||
begin F20 "Загрузка растра GeoTIFF (600×400, EPSG:4326, Float32)"
|
||||
http POST /datasets -F "file=@$NDVI" -F file_type=raster -F "category_id=$CAT_ECO" -F "name=NDVI СКО" -F unit=NDVI
|
||||
need_code 201; DS_NDVI="$(jb .id)"; track_ds "$DS_NDVI"; s0="$(jb .status)"
|
||||
wait_ds "$DS_NDVI" ready; need_eq "$WS" ready status
|
||||
end "201, status $s0 → $WS"
|
||||
|
||||
begin F21 "Растр: статус, bbox"
|
||||
http GET "/datasets/$DS_NDVI"
|
||||
need_code 200; need_eq "$(jb .status)" ready status
|
||||
need_has "$(jb .cog_storage_key)" "/cog/ndvi_nko.tif" cog_storage_key
|
||||
bbox="$(jb '.bbox|map(.*1000|round/1000)|join(", ")')"
|
||||
need_eq "$bbox" "68.5, 51.5, 74.5, 55.5" bbox
|
||||
end "200; ready, cog_storage_key …/cog/ndvi_nko.tif, bbox [$bbox]"
|
||||
|
||||
begin F22 "Получение COG и проверка LAYOUT=COG"
|
||||
ctype="$(curl -sS -o "$TMP/ndvi.cog.tif" -w '%{http_code} %{content_type}' "$BASE_URL/datasets/$DS_NDVI.cog" || echo 000)"
|
||||
need_eq "$ctype" "200 image/tiff" "код и Content-Type"
|
||||
ri="$("$PYTHON" "$HERE/lib/helpers.py" rinfo "$TMP/ndvi.cog.tif" 2>&1 || true)"
|
||||
need_eq "$(echo "$ri" | jq -r '"\(.driver) \(.width)x\(.height) EPSG:\(.epsg) \(.layout) \(.compression)"' 2>/dev/null)" "GTiff 600x400 EPSG:4326 COG DEFLATE" "формат"
|
||||
end "$ctype, $(wc -c < "$TMP/ndvi.cog.tif" | tr -d ' ') байт; $(echo "$ri" | jq -r '"\(.driver), \(.width)×\(.height), EPSG:\(.epsg), LAYOUT=\(.layout), \(.compression)"' 2>/dev/null)"
|
||||
|
||||
begin F23 "Скачивание оригинала /download"
|
||||
dl="$(curl -sS -o "$TMP/ndvi.dl.tif" -w '%{http_code}' "$BASE_URL/datasets/$DS_NDVI/download" || echo 000)"
|
||||
need_eq "$dl" 200 "HTTP"
|
||||
h1="$(sha256 "$NDVI")"; h2="$(sha256 "$TMP/ndvi.dl.tif")"
|
||||
need_eq "$h2" "$h1" "SHA-256"
|
||||
end "$dl, $(wc -c < "$TMP/ndvi.dl.tif" | tr -d ' ') байт; SHA-256 совпадает (${h1:0:8}…${h1: -5})"
|
||||
|
||||
begin F24 "Загрузка SLD для набора"
|
||||
http GET "/datasets/$DS_NDVI/sld"; need_code 404; c0="$CODE"
|
||||
http POST "/datasets/$DS_NDVI/sld" -F "file=@$DATA/style.sld"; need_code 200; need_eq "$(jb .sld_filename)" style.sld sld_filename; c1="$CODE"
|
||||
http GET "/datasets/$DS_NDVI/sld"; need_code 200; need_has "$(cat "$BODY")" "NDVI gradient" "SLD"
|
||||
end "до загрузки GET /sld → $c0; POST → $c1, sld_filename style.sld; GET /sld → $CODE («NDVI gradient»)"
|
||||
|
||||
begin F25 "Общий SLD группы и наследование набором"
|
||||
http POST "/dataset-groups/$GRP/sld" -F "file=@$DATA/group_style.sld"; need_code 200; c1="$CODE"
|
||||
http GET "/dataset-groups/$GRP/sld"; need_code 200; need_has "$(cat "$BODY")" "Group fallback style" "SLD группы"
|
||||
http GET "/datasets/$DS_POP/sld"; need_code 200; need_has "$(cat "$BODY")" "Group fallback style" "SLD набора без собственного"
|
||||
end "POST группы → $c1; GET /sld группы и набора без собственного SLD → «Group fallback style»"
|
||||
|
||||
begin N06 "SLD: недопустимое расширение файла"
|
||||
http POST "/datasets/$DS_NDVI/sld" -F "file=@$DATA/style.txt"
|
||||
need_code 422; need_has "$(jb .error)" "sld file must have one of these extensions" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
|
||||
begin F26 "PATCH набора: dg_id и период"
|
||||
http PATCH "/datasets/$DS_NDVI" "${JSON[@]}" -d "{\"dg_id\":\"$GRP\",\"start_date\":\"2026-01-01\",\"end_date\":\"2026-03-31\",\"description\":\"NDVI за I квартал 2026\"}"
|
||||
need_code 200; need_eq "$(jb .dg_id)" "$GRP" dg_id; need_eq "$(jb .start_date)" 2026-01-01 start_date
|
||||
need_eq "$(jb .end_date)" 2026-03-31 end_date; need_eq "$(jb .description)" "NDVI за I квартал 2026" description
|
||||
end "200; dg_id, start_date 2026-01-01, end_date 2026-03-31, description обновлены"
|
||||
|
||||
begin F27 "PUT набора: полная замена метаданных"
|
||||
http PUT "/datasets/$DS_PLOTS" "${JSON[@]}" -d "{\"category_id\":\"$CAT_ECO\",\"name\":\"Участки СКО (ред.)\",\"description\":\"Полигоны участков\",\"unit\":\"га\",\"dg_id\":\"$GRP\",\"start_date\":\"2025-01-01\",\"end_date\":\"2025-12-31\"}"
|
||||
need_code 200; need_eq "$(jb .name)" "Участки СКО (ред.)" name; need_eq "$(jb .unit)" "га" unit
|
||||
need_eq "$(jb .dg_id)" "$GRP" dg_id; need_eq "$(jb .end_date)" 2025-12-31 end_date
|
||||
end "200; name «Участки СКО (ред.)», unit га, dg_id, период 2025"
|
||||
|
||||
begin N07 "PATCH: end_date раньше start_date"
|
||||
http PATCH "/datasets/$DS_PLOTS" "${JSON[@]}" -d '{"start_date":"2025-12-31","end_date":"2025-01-01"}'
|
||||
need_code 422; need_has "$(jb .error)" 'end_date "2025-01-01" is before start_date "2025-12-31"' сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N08 "PATCH: нередактируемое поле file_type"
|
||||
http PATCH "/datasets/$DS_PLOTS" "${JSON[@]}" -d '{"file_type":"raster"}'
|
||||
need_code 422; need_eq "$(jb .error)" "unknown field: file_type" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N09 "PUT: отсутствуют обязательные поля"
|
||||
http PUT "/datasets/$DS_PLOTS" "${JSON[@]}" -d '{"name":"Только имя"}'
|
||||
need_code 422
|
||||
need_eq "$(jb '[.errors.CategoryID,.errors.Description,.errors.Unit]|join(",")')" "is required,is required,is required" errors
|
||||
end "422 CategoryID, Description, Unit «is required»"
|
||||
|
||||
begin F28 "Свежайший набор группы /latest (по max end_date, фильтры file_type, status)"
|
||||
http GET "/dataset-groups/$GRP/latest"; need_code 200; need_eq "$(jb .id)" "$DS_NDVI" "latest"; l1="$(jb .end_date)"
|
||||
http GET "/dataset-groups/$GRP/latest?file_type=vector_with_kato"; need_code 200; need_eq "$(jb .id)" "$DS_POP" "latest vector_with_kato"; l2="$(jb .end_date)"
|
||||
http GET "/dataset-groups/$GRP/latest?status=failed"; need_code 404
|
||||
end "200; NDVI (end_date $l1); с file_type=vector_with_kato — набор населения ($l2); status=failed → $CODE"
|
||||
|
||||
begin F29 "Обновление группы"
|
||||
http PUT "/dataset-groups/$GRP" "${JSON[@]}" -d "{\"code\":\"$P-population-nko\",\"name\":\"Население и NDVI СКО\",\"description\":\"Приёмочный тест\"}"
|
||||
need_code 200; need_eq "$(jb .name)" "Население и NDVI СКО" name
|
||||
end "200; name «Население и NDVI СКО»"
|
||||
|
||||
begin F30 "Удаление группы, на которую ссылаются наборы"
|
||||
http DELETE "/dataset-groups/$GRP"
|
||||
need_code 409
|
||||
end "409 «$(jb .error)»"
|
||||
|
||||
begin F31 "Список групп с фильтром code"
|
||||
http GET "/dataset-groups?code=$P-population-nko"
|
||||
need_code 200; need_eq "$(jb .total)" 1 total; need_eq "$(jb '.data[0].id')" "$GRP" группа
|
||||
end "200; total $(jb .total)"
|
||||
|
||||
begin F32 "Удаление пустой группы"
|
||||
http POST /dataset-groups "${JSON[@]}" -d "{\"code\":\"$P-empty\",\"name\":\"Пустая\"}"
|
||||
need_code 201; eg="$(jb .id)"; c1="$CODE"
|
||||
http DELETE "/dataset-groups/$eg"; need_code 204
|
||||
[ "$CODE" = 204 ] || GRPS="$eg $GRPS"
|
||||
end "$c1 → $CODE"
|
||||
|
||||
begin N10 "Группа: код не slug («Bad Code!»)"
|
||||
http POST /dataset-groups "${JSON[@]}" -d '{"code":"Bad Code!","name":"Плохой код"}'
|
||||
need_code 422; need_eq "$(jb .errors.Code)" "is invalid" errors.Code
|
||||
[ "$CODE" = 201 ] && GRPS="$(jb .id) $GRPS"
|
||||
end "422 $(jbc)"
|
||||
|
||||
begin F33 "Список наборов: пагинация"
|
||||
http GET "/datasets?category_id=$CAT_ECO&page_size=2&page=1"
|
||||
need_code 200; need_eq "$(jb .total)" 4 total; need_eq "$(jb .page_size)" 2 page_size; need_eq "$(jb .total_pages)" 2 total_pages
|
||||
p1="$(jb '[.data[].id]|join(" ")')"
|
||||
http GET "/datasets?category_id=$CAT_ECO&page_size=2&page=2"
|
||||
need_code 200; p2="$(jb '[.data[].id]|join(" ")')"
|
||||
all="$(printf '%s\n' $p1 $p2 | sort -u | wc -l | tr -d ' ')"
|
||||
need_eq "$all" 4 "уникальных записей на 2 страницах"
|
||||
end "200; total 4, page_size 2, total_pages 2; страницы 1 и 2 без пересечений"
|
||||
|
||||
begin F34 "Фильтры dg_id + end_date_from, file_type + status, category_id + start_date_to"
|
||||
http GET "/datasets?dg_id=$GRP&end_date_from=2025-01-01"; need_code 200; n1="$(jb .total)"
|
||||
need_eq "$(jb '[.data[].id]|sort|join(" ")')" "$(printf '%s\n' "$DS_PLOTS" "$DS_NDVI" | sort | tr '\n' ' ' | sed 's/ $//')" "dg_id+end_date_from"
|
||||
http GET "/datasets?category_id=$CAT_ECO&file_type=vector_with_kato&status=ready"; need_code 200; n2="$(jb .total)"
|
||||
need_eq "$(jb '[.data[].id]|sort|join(" ")')" "$(printf '%s\n' "$DS_POP" "$DS_LIVE" | sort | tr '\n' ' ' | sed 's/ $//')" "file_type+status"
|
||||
http GET "/datasets?category_id=$CAT_ECO&start_date_to=2024-01-01"; need_code 200; n3="$(jb .total)"
|
||||
need_eq "$(jb '[.data[].id]|join(" ")')" "$DS_POP" "category_id+start_date_to"
|
||||
end "200; $n1 / $n2 / $n3 записи соответственно, состав соответствует условиям"
|
||||
|
||||
begin N11 "Фильтр: некорректная дата 2025-31-12"
|
||||
http GET "/datasets?start_date_from=2025-31-12"
|
||||
need_code 400; need_eq "$(jb .error)" "start_date_from must be YYYY-MM-DD" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
begin N12 "Пагинация: page_size=0"
|
||||
http GET "/datasets?page_size=0"
|
||||
need_code 400; need_eq "$(jb .error)" "invalid page_size" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
begin N13 "Фильтр: file_type=shapefile"
|
||||
http GET "/datasets?file_type=shapefile"
|
||||
need_code 400; need_has "$(jb .error)" "file_type must be one of" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
|
||||
# ====================================================================== 5.4
|
||||
printf 'just text\n' > "$TMP/notes.txt"
|
||||
printf 'this is not a tiff\n' > "$TMP/fake.tif"
|
||||
printf 'not json at all\n' > "$TMP/bad.geojson"
|
||||
up() { http POST /datasets "$@"; [ "$CODE" = 201 ] && track_ds "$(jb .id)"; return 0; }
|
||||
negup() { # ID desc expected-code expected-substring curl-args...
|
||||
local id="$1" desc="$2" ec="$3" sub="$4"; shift 4
|
||||
begin "$id" "$desc"; up "$@"; need_code "$ec"; need_has "$(jb .error)" "$sub" сообщение
|
||||
end "$CODE «$(jb .error)»"
|
||||
}
|
||||
negup N14 "Загрузка: расширение .txt" 422 'extension ".txt" is not allowed for file_type "vector"' \
|
||||
-F "file=@$TMP/notes.txt" -F file_type=vector -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N15 "Загрузка: .tif без TIFF-сигнатуры" 422 "file is not a valid TIFF/BigTIFF GeoTIFF" \
|
||||
-F "file=@$TMP/fake.tif" -F file_type=raster -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N16 "Загрузка: .geojson не JSON" 422 "file is not valid GeoJSON" \
|
||||
-F "file=@$TMP/bad.geojson" -F file_type=vector -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N17 "Загрузка: неизвестный file_type" 422 "file_type must be one of: vector_with_kato, vector, raster" \
|
||||
-F "file=@$DATA/plots_nko.geojson" -F file_type=shapefile -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N18 "Загрузка: .tif для типа vector" 422 'extension ".tif" is not allowed for file_type "vector"' \
|
||||
-F "file=@$NDVI" -F file_type=vector -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N19 "Загрузка: нет файла" 400 "file is required" \
|
||||
-F file_type=vector -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N20 "Загрузка: нет category_id" 422 "category_id must be a valid UUID" \
|
||||
-F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F name=x
|
||||
|
||||
begin N21 "Загрузка без name (поле необязательно по OpenAPI)"
|
||||
up -F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F "category_id=$CAT_SOIL"
|
||||
need_code 201; DS_N21="$(jb .id)"; N21_KEY="$(jb .storage_key)"; need_eq "$(jb .name)" plots_nko.geojson name
|
||||
end "201; name = имя файла ($(jb .name))"
|
||||
|
||||
negup N22 "Загрузка: end_date раньше start_date" 422 'end_date "2025-01-01" is before start_date "2025-12-31"' \
|
||||
-F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F "category_id=$CAT_SOIL" -F name=x -F start_date=2025-12-31 -F end_date=2025-01-01
|
||||
negup N23 "Загрузка: несуществующая категория" 422 "category does not exist" \
|
||||
-F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F "category_id=00000000-0000-4000-8000-000000000000" -F name=x
|
||||
negup N24 "Загрузка: невалидный JSON в meta" 422 "meta must be valid JSON" \
|
||||
-F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F "category_id=$CAT_SOIL" -F name=x -F 'meta={bad json'
|
||||
begin N25 "Загрузка: не multipart"
|
||||
http POST /datasets "${JSON[@]}" -d '{"file_type":"vector"}'
|
||||
need_code 400; need_eq "$(jb .error)" "request must be multipart/form-data" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
|
||||
begin N26 "Несуществующие id на 10 эндпоинтах"
|
||||
X=7d3c9a52-1f4e-4b8a-9c2d-5e6f7a8b9c0d; codes=""
|
||||
for p in "/datasets/$X" "/datasets/$X.geojson" "/datasets/$X.cog" "/datasets/$X/download" "/datasets/$X/observations" \
|
||||
"/datasets/$X/sld" "/categories/$X" "/dataset-groups/$X" "/dataset-groups/$X/latest" "/landscape/runs/$X"; do
|
||||
http GET "$p"; codes="$codes $CODE"; [ "$CODE" = 404 ] || bad "$p → $CODE"
|
||||
done
|
||||
end "404 на всех 10 эндпоинтах:$codes"
|
||||
|
||||
begin N27 "Некорректный формат id (4 эндпоинта)"
|
||||
codes=""
|
||||
for p in /datasets/not-a-uuid /categories/not-a-uuid /dataset-groups/not-a-uuid /landscape/runs/not-a-uuid; do
|
||||
http GET "$p"; codes="$codes $CODE"; [ "$CODE" = 400 ] || bad "$p → $CODE"; [ "$(jb .error)" = "invalid id" ] || bad "$p: $(jb .error)"
|
||||
done
|
||||
end "400 «invalid id» на всех 4:$codes"
|
||||
|
||||
begin N28 "Битый JSON"
|
||||
http POST /categories "${JSON[@]}" -d '{"code":"x","name":'
|
||||
need_code 400; need_eq "$(jb .error)" "invalid request body" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
begin N29 "Мусорное тело 200 КБ"
|
||||
head -c 204800 /dev/urandom > "$TMP/garbage.bin"
|
||||
http POST /categories "${JSON[@]}" --data-binary "@$TMP/garbage.bin"
|
||||
need_code 400; need_eq "$(jb .error)" "invalid request body" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
begin N30 "Неизвестное поле в JSON"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-unknown\",\"name\":\"x\",\"color\":\"red\"}"
|
||||
need_code 400; need_eq "$(jb .error)" "invalid request body" сообщение
|
||||
[ "$CODE" = 201 ] && CATS="$(jb .id) $CATS"
|
||||
end "400 «$(jb .error)»"
|
||||
begin N37 "JSON-тело 2 200 048 байт (> 1 МиБ)"
|
||||
{ printf '{"code":"%s-big","name":"' "$P"; head -c $((2200048 - 25 - ${#P})) /dev/zero | tr '\0' 'a'; printf '"}'; } > "$TMP/big.json"
|
||||
http POST /categories "${JSON[@]}" --data-binary "@$TMP/big.json"
|
||||
need_code 400; need_eq "$(jb .error)" "invalid request body" сообщение
|
||||
[ "$CODE" = 201 ] && CATS="$(jb .id) $CATS"
|
||||
end "400 «$(jb .error)» (тело $(wc -c < "$TMP/big.json" | tr -d ' ') байт)"
|
||||
|
||||
runreq() { http POST /landscape/runs "${JSON[@]}" -d "$1"; [ "$CODE" = 202 ] && RUNS="$RUNS $(jb .id)"; return 0; }
|
||||
begin N31 "ЛП: from_stage=7"
|
||||
runreq '{"from_stage":7}'; need_code 422; need_eq "$(jb .errors.FromStage)" "must be 3 or less" errors.FromStage
|
||||
end "422 $(jbc)"
|
||||
begin N32 "ЛП: tie_priority не перестановка"
|
||||
runreq '{"params":{"tie_priority":[1,1,2,3,4]}}'; need_code 422; need_has "$(jb .error)" "tie_priority must be a permutation of 1..5" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N33 "ЛП: override с некорректным UUID"
|
||||
runreq '{"overrides":{"lp-z-lcr":"not-a-uuid"}}'; need_code 422; need_eq "$(jb '.errors["Overrides[lp-z-lcr]"]')" "must be a valid UUID" errors
|
||||
end "422 $(jbc)"
|
||||
begin N34 "ЛП: неизвестный слот"
|
||||
http POST /landscape/slots/lp-unknown-slot -F "file=@$NDVI"; need_code 404
|
||||
[ "$CODE" = 201 ] && track_ds "$(jb .id)"
|
||||
end "404 «$(jb .error)»"
|
||||
begin N35 "ЛП: не-GeoTIFF в слот"
|
||||
http POST /landscape/slots/lp-z-lcr -F "file=@$DATA/plots_nko.geojson"; need_code 422
|
||||
need_has "$(jb .error)" 'extension ".geojson" is not allowed for file_type "raster"' сообщение
|
||||
[ "$CODE" = 201 ] && track_ds "$(jb .id)"
|
||||
end "422 «$(jb .error)»"
|
||||
begin N36 "ЛП: window_size=100"
|
||||
runreq '{"params":{"window_size":100}}'; need_code 422; need_has "$(jb .error)" "window_size must be in 256..8192" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
|
||||
begin N38 "Загрузка 100 МБ мусора с сигнатурой TIFF"
|
||||
{ printf 'II*\000'; head -c 104857600 /dev/urandom; } > "$TMP/garbage.tif"
|
||||
up -F "file=@$TMP/garbage.tif" -F file_type=raster -F "category_id=$CAT_SOIL" -F "name=$P garbage"
|
||||
need_code 201; DS_G="$(jb .id)"; up_time="$TIME"
|
||||
wait_ds "$DS_G" ready; need_eq "$WS" failed status
|
||||
http GET "/datasets/$DS_G"; pe="$(jb .parse_error)"; need_has "$pe" gdal_translate parse_error
|
||||
http GET /readyz; need_code 200; need_eq "$(jb .ready)" true "/readyz ready"
|
||||
rm -f "$TMP/garbage.tif"
|
||||
end "201 (104 857 604 байт за ${up_time} с) → status $WS, parse_error от gdal_translate; /readyz 200, сервис доступен"
|
||||
|
||||
begin F35 "Удаление набора (строка и объект в S3)"
|
||||
http DELETE "/datasets/$DS_N21"; need_code 204; c1="$CODE"
|
||||
http GET "/datasets/$DS_N21"; need_code 404; c2="$CODE"
|
||||
detail="$c1; GET → $c2"
|
||||
if [ "$LOCAL_CHECKS" = 1 ]; then
|
||||
prefix="${N21_KEY%/*}"
|
||||
n="$(docker exec "$MINIO_C" sh -c "MC_CONFIG_DIR=/tmp/.mc MC_HOST_local=http://\$MINIO_ROOT_USER:\$MINIO_ROOT_PASSWORD@127.0.0.1:9000 mc ls --recursive local/geofiles/$prefix/ 2>/dev/null | wc -l" | tr -d ' ' || echo "?")"
|
||||
need_eq "$n" 0 "объектов в префиксе MinIO"
|
||||
detail="$detail; объектов в префиксе MinIO $prefix/ — $n"
|
||||
else
|
||||
detail="$detail; проверка MinIO пропущена (LOCAL_CHECKS=0)"
|
||||
fi
|
||||
end "$detail"
|
||||
|
||||
# ====================================================================== 5.5
|
||||
psql_q() { docker exec -i "$PG_C" psql -U gis -d gis -At -F '|' -c "$1"; }
|
||||
begin F36 "Геометрии наборов в PostGIS: тип, SRID, ST_IsValid"
|
||||
if [ "$LOCAL_CHECKS" = 1 ]; then
|
||||
out="$(psql_q "SELECT id, ST_GeometryType(geometry), ST_SRID(geometry), ST_IsValid(geometry) FROM datasets WHERE id IN ('$DS_PLOTS','$DS_POP','$DS_LIVE','$DS_NDVI')" 2>&1 || true)"
|
||||
echo "$out" > "$RESULTS/F36.txt"
|
||||
for pair in "$DS_PLOTS:ST_MultiPolygon" "$DS_POP:ST_MultiPoint" "$DS_LIVE:ST_MultiPoint" "$DS_NDVI:ST_Polygon"; do
|
||||
id="${pair%%:*}"; want="${pair#*:}"
|
||||
row="$(echo "$out" | grep "^$id|" || true)"
|
||||
need_eq "$(echo "$row" | cut -d'|' -f2-4)" "$want|4326|t" "геометрия $id"
|
||||
done
|
||||
end "участки ST_MultiPolygon; население и КРС ST_MultiPoint; NDVI ST_Polygon; у всех SRID 4326, valid = t"
|
||||
else skip "LOCAL_CHECKS=0 (нет доступа к контейнеру PostGIS)"; fi
|
||||
|
||||
begin F37 "Наблюдения dataset_observations соединяются с districts по КАТО"
|
||||
if [ "$LOCAL_CHECKS" = 1 ]; then
|
||||
out="$(psql_q "SELECT count(*), count(DISTINCT o.kato_code), count(*) FILTER (WHERE d.kato IS NULL) FROM dataset_observations o LEFT JOIN districts d ON d.kato = o.kato_code WHERE o.dataset_id IN ('$DS_POP','$DS_LIVE')" 2>&1 || true)"
|
||||
need_eq "$out" "84|14|0" "наблюдений|кодов КАТО|сирот"
|
||||
kz="$(psql_q "SELECT d.name || ' (' || d.kato || ') — ' || string_agg(o.value::text, ' / ' ORDER BY o.start_date) FROM dataset_observations o JOIN districts d ON d.kato = o.kato_code WHERE o.dataset_id = '$DS_POP' AND o.kato_code = '595000000' GROUP BY d.name, d.kato" 2>&1 || true)"
|
||||
end "84 наблюдения, 14 кодов КАТО, orphan_kato 0 ($out); $kz"
|
||||
else skip "LOCAL_CHECKS=0 (нет доступа к контейнеру PostGIS)"; fi
|
||||
|
||||
begin F38 "Объекты в бакете geofiles MinIO"
|
||||
if [ "$LOCAL_CHECKS" = 1 ]; then
|
||||
keys=""
|
||||
for d in "$DS_PLOTS" "$DS_POP" "$DS_LIVE" "$DS_NDVI"; do
|
||||
http GET "/datasets/$d"; keys="$keys $(jb '[.storage_key, .cog_storage_key, .sld_storage_key] | map(select(. != null)) | join(" ")')"
|
||||
done
|
||||
http GET "/dataset-groups/$GRP"; keys="$keys $(jb .sld_storage_key)"
|
||||
found=0; total=0
|
||||
for k in $keys; do
|
||||
total=$((total + 1))
|
||||
if docker exec "$MINIO_C" sh -c "MC_CONFIG_DIR=/tmp/.mc MC_HOST_local=http://\$MINIO_ROOT_USER:\$MINIO_ROOT_PASSWORD@127.0.0.1:9000 mc stat local/geofiles/$k" >/dev/null 2>&1; then
|
||||
found=$((found + 1))
|
||||
else bad "нет объекта $k"; fi
|
||||
done
|
||||
du="$(docker exec "$MINIO_C" sh -c 'MC_CONFIG_DIR=/tmp/.mc MC_HOST_local=http://$MINIO_ROOT_USER:$MINIO_ROOT_PASSWORD@127.0.0.1:9000 mc du local/geofiles' 2>/dev/null | tr -s ' \t' ' ' || true)"
|
||||
[ "$total" -eq 7 ] || bad "ожидалось 7 ключей (4 исходных файла, COG, SLD набора, SLD группы), получено $total"
|
||||
end "$found/$total объектов найдено: plots_nko.geojson, population_kato.geojson, livestock_kato.gpkg, ndvi_nko.tif, cog/ndvi_nko.tif, sld/style.sld, groups/…/sld/group_style.sld; бакет: $du"
|
||||
else skip "LOCAL_CHECKS=0 (нет доступа к контейнеру MinIO)"; fi
|
||||
|
||||
# ====================================================================== 5.6 ЛП
|
||||
LPIN="$TMP/lpin"; LPCACHE="$TMP/lpcache"; mkdir -p "$LPCACHE"
|
||||
"$PYTHON" "$HERE/lp_gen.py" --out "$LPIN" > "$RESULTS/lp_gen.txt"
|
||||
SLOTS="lp-z-biodiversity lp-z-lcr lp-z-soil-protection lp-z-wr lp-u-surf-water lp-u-soil lp-u-crop lp-u-veg lp-p-irrigation lp-p-snow lp-p-crop lp-p-pasture lp-p-tourism"
|
||||
slot_id_file() { echo "$TMP/slot-$1"; }
|
||||
|
||||
begin L02 "Загрузка 13 входных растров в слоты POST /landscape/slots/{code}"
|
||||
codes=""
|
||||
for s in $SLOTS; do
|
||||
http POST "/landscape/slots/$s" -F "file=@$LPIN/$s.tif" -F "name=$P $s" -F start_date=2025-01-01 -F end_date=2025-12-31
|
||||
codes="$codes $CODE"; [ "$CODE" = 201 ] || bad "$s → $CODE: $(snippet)"
|
||||
id="$(jb .id)"; echo "$id" > "$(slot_id_file "$s")"; track_ds "$id"
|
||||
done
|
||||
for s in $SLOTS; do wait_ds "$(cat "$(slot_id_file "$s")")" ready; [ "$WS" = ready ] || bad "$s: status $WS"; done
|
||||
http GET /landscape/slots
|
||||
for s in $SLOTS; do
|
||||
got="$(jq -r --arg s "$s" '.[]|select(.code==$s)|"\(.latest.id) \(.latest.status)"' "$BODY")"
|
||||
need_eq "$got" "$(cat "$(slot_id_file "$s")") ready" "слот $s"
|
||||
done
|
||||
end "201 для всех 13 слотов; GET /landscape/slots — у всех слотов latest = загруженный растр, status ready"
|
||||
|
||||
begin L03 "Запуск полного прогона {\"from_stage\":0}"
|
||||
runreq '{"from_stage":0}'; need_code 202; RUN1="$(jb .id)"; need_eq "$(jb .status)" queued status
|
||||
need_eq "$(jq -S -c .params "$BODY")" '{"tie_priority":[1,2,3,4,5],"v1_min_s_e12":3,"v1_min_s_e3":4,"v2_min_s_e3":3,"v2_min_s_e4":5,"window_size":2048}' params
|
||||
end "202; run ${RUN1:0:8} поставлен в очередь, params по умолчанию (window_size 2048, tie_priority [1..5], пороги 3/4/3/5)"
|
||||
|
||||
begin L04 "Завершение прогона"
|
||||
wait_run "$RUN1"; need_eq "$RS" succeeded status; ns="$(node_statuses "$RUN1")"
|
||||
need_eq "$ns" "00:succeeded 01:succeeded 02:succeeded 03:succeeded 10:succeeded 20:succeeded 21:succeeded 30:succeeded 31:succeeded" узлы
|
||||
need_eq "$(jq -r .error "$TMP/run-$RUN1.json")" null error
|
||||
end "$RS за $("$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run-$RUN1.json"); error null; все 9 узлов succeeded"
|
||||
|
||||
begin L05 "Результаты GET /landscape/results"
|
||||
outs="$(run_outputs "$RUN1")"
|
||||
# Results list only ready layers; the COG conversion of the outputs is asynchronous.
|
||||
for id in $(jq -r '.[].dataset_id' <<< "$outs"); do wait_ds "$id" ready; [ "$WS" = ready ] || bad "выход $id: status $WS"; done
|
||||
http GET /landscape/results; need_code 200; cp "$BODY" "$TMP/results.json"
|
||||
need_eq "$(jb 'length')" 15 "число групп"
|
||||
for g in $(jq -r '.[].group' <<< "$outs"); do
|
||||
want="$(jq -r --arg g "$g" '.[]|select(.group==$g)|.dataset_id' <<< "$outs")"
|
||||
got="$(jq -r --arg g "$g" '.[]|select(.group==$g)|"\(.latest.id) \(.latest.status) \(.cog_url)"' "$TMP/results.json")"
|
||||
need_eq "$got" "$want ready /datasets/$want.cog" "результат $g"
|
||||
done
|
||||
first="$(jq -r '.[0].dataset_id' <<< "$outs")"
|
||||
http GET "/datasets/$first"; need_eq "$(jb .automated)" true automated; need_eq "$(jb .meta.lp_run_id)" "$RUN1" meta.lp_run_id
|
||||
[ "$(jb .meta.lp_node)" != null ] || bad "meta.lp_node"; [ "$(jb '.meta.inputs|length')" -ge 1 ] 2>/dev/null || bad "meta.inputs"
|
||||
fmt_ok=0
|
||||
for id in $(jq -r '.[].dataset_id' <<< "$outs"); do
|
||||
curl -fsS -o "$LPCACHE/$id.cog.tif" "$BASE_URL/datasets/$id.cog" || rm -f "$LPCACHE/$id.cog.tif"
|
||||
ri="$("$PYTHON" "$HERE/lib/helpers.py" rinfo "$LPCACHE/$id.cog.tif" 2>/dev/null || echo '{}')"
|
||||
f="$(jq -r '"\(.width)x\(.height) EPSG:\(.epsg) \(.layout) \(.dtype) \(.nodata|floor)"' <<< "$ri" 2>/dev/null || true)"
|
||||
if [ "$f" = "3000x2000 EPSG:32642 COG int16 -9999" ]; then fmt_ok=$((fmt_ok + 1)); else bad "COG $id: $f"; fi
|
||||
done
|
||||
end "200; 15 групп (узлы 00…31), все ready, cog_url /datasets/{id}.cog; выходной набор: automated true, meta.lp_run_id, lp_node, inputs. $fmt_ok/15 COG: 3000×2000, EPSG:32642, LAYOUT=COG, Int16, NoData −9999"
|
||||
|
||||
begin L07 "Попиксельная сверка 15 выходов с эталонной реализацией"
|
||||
if "$PYTHON" "$HERE/lp_verify.py" --base-url "$BASE_URL" --run "$RUN1" --cache "$LPCACHE" > "$RESULTS/L07_lp_verify.txt" 2>&1; then :; else bad "lp_verify: $(tail -n 3 "$RESULTS/L07_lp_verify.txt" | tr '\n' ' ')"; fi
|
||||
tot="$(grep 'TOTAL mismatching' "$RESULTS/L07_lp_verify.txt" || echo 'нет итога')"
|
||||
end "15 выходов × 6 000 000 пикселей; «$tot» (журнал: results/functional-$STAMP/L07_lp_verify.txt)"
|
||||
|
||||
begin L08 "Выборочная проверка 9 пикселей (Z_int, U_int, E, C_k, V1–V3)"
|
||||
PTS="20,20;1450,1000;1575,1000;350,350;520,320;500,1500;1000,700;2200,1500;2900,1900"
|
||||
if "$PYTHON" "$HERE/lp_verify.py" --base-url "$BASE_URL" --run "$RUN1" --cache "$LPCACHE" --points "$PTS" > "$RESULTS/L08_points.txt" 2>&1; then :; else bad "$(tail -n 1 "$RESULTS/L08_points.txt")"; fi
|
||||
end "$(tail -n 1 "$RESULTS/L08_points.txt"): NoData (20,20), дыра NoData в U crop (1450,1000), crop и veg NoData (1575,1000), P snow = 0 и 6"
|
||||
|
||||
begin L09 "Повторный прогон {} без изменений входов"
|
||||
runreq '{}'; need_code 202; RUN2="$(jb .id)"; wait_run "$RUN2"; need_eq "$RS" succeeded status
|
||||
ns="$(node_statuses "$RUN2")"
|
||||
need_eq "$ns" "00:skipped 01:skipped 02:skipped 03:skipped 10:skipped 20:skipped 21:skipped 30:skipped 31:skipped" узлы
|
||||
need_eq "$(run_outputs "$RUN2")" "$(run_outputs "$RUN1")" "выходные наборы"
|
||||
end "202; run ${RUN2:0:8} $RS за $("$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run-$RUN2.json"); все 9 узлов skipped, выходные наборы совпадают с первым прогоном"
|
||||
|
||||
begin L10 "Прогон {\"from_stage\":3}"
|
||||
runreq '{"from_stage":3}'; need_code 202; RUN3="$(jb .id)"; wait_run "$RUN3"; need_eq "$RS" succeeded status
|
||||
ns="$(node_statuses "$RUN3")"
|
||||
need_eq "$ns" "00:skipped 01:skipped 02:skipped 03:skipped 10:skipped 20:skipped 21:skipped 30:succeeded 31:succeeded" узлы
|
||||
new="$(jq -r '[.nodes[]|select(.node_code=="30" or .node_code=="31")|.outputs[]?.dataset_id]|join(" ")' "$TMP/run-$RUN3.json" || true)"
|
||||
for id in $new; do run_outputs "$RUN1" | grep -q "$id" && bad "узел стадии 3 переиспользовал $id"; done
|
||||
end "202; run ${RUN3:0:8} $RS за $("$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run-$RUN3.json"); узлы 00–21 skipped, узлы 30 и 31 succeeded (новые выходные наборы)"
|
||||
|
||||
begin L11 "Прогон с параметрами tie_priority [5,4,3,2,1], пороги 4/5/4/4"
|
||||
runreq '{"params":{"tie_priority":[5,4,3,2,1],"v1_min_s_e12":4,"v1_min_s_e3":5,"v2_min_s_e3":4,"v2_min_s_e4":4}}'
|
||||
need_code 202; RUN4="$(jb .id)"; wait_run "$RUN4"; need_eq "$RS" succeeded status
|
||||
ns="$(node_statuses "$RUN4")"
|
||||
need_eq "$ns" "00:skipped 01:skipped 02:skipped 03:skipped 10:skipped 20:skipped 21:skipped 30:succeeded 31:skipped" узлы
|
||||
end "202; run ${RUN4:0:8} $RS за $("$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run-$RUN4.json"); пересчитан только узел 30 (V1–V3), остальные skipped"
|
||||
|
||||
begin L12 "Сверка V1–V3 с пользовательскими параметрами"
|
||||
wait_outputs "$RUN4"
|
||||
if "$PYTHON" "$HERE/lp_verify.py" --base-url "$BASE_URL" --run "$RUN4" --cache "$LPCACHE" > "$RESULTS/L12_lp_verify.txt" 2>&1; then :; else bad "lp_verify: $(tail -n 3 "$RESULTS/L12_lp_verify.txt" | tr '\n' ' ')"; fi
|
||||
v="$(grep -E '^lp-v[123]-' "$RESULTS/L12_lp_verify.txt" | awk '{printf "%s mismatch %s; ", $1, $6}' || true)"
|
||||
end "$v; «$(grep 'TOTAL mismatching' "$RESULTS/L12_lp_verify.txt" || echo 'нет итога')» (все 15 выходов)"
|
||||
|
||||
begin L13 "Невыровненный растр (1500×1000, 200 м) в слоте lp-u-veg"
|
||||
http POST /landscape/slots/lp-u-veg -F "file=@$LPIN/misaligned-lp-u-veg.tif" -F "name=$P misaligned" -F start_date=2025-01-01 -F end_date=2025-12-31
|
||||
need_code 201; DS_MIS="$(jb .id)"; track_ds "$DS_MIS"; wait_ds "$DS_MIS" ready; need_eq "$WS" ready "status растра"
|
||||
runreq '{}'; need_code 202; RUN5="$(jb .id)"; wait_run "$RUN5"; need_eq "$RS" failed status
|
||||
err="$(jq -r .error "$TMP/run-$RUN5.json" || true)"
|
||||
need_has "$err" "input lp-u-veg grid (1500x1000" "сообщение"; need_has "$err" "inputs must be aligned" "сообщение"
|
||||
later="$(jq -r '[.nodes[]|select(.stage>=1)|.status]|unique|join(",")' "$TMP/run-$RUN5.json" || true)"
|
||||
need_eq "$later" pending "узлы стадий 1–3"
|
||||
end "run ${RUN5:0:8} $RS: «${err:0:160}…»; последующие узлы не выполняются ($later)"
|
||||
|
||||
begin L14 "Восстановление: удаление растра, повторный прогон"
|
||||
http DELETE "/datasets/$DS_MIS"; need_code 204; c1="$CODE"
|
||||
http GET /landscape/slots
|
||||
need_eq "$(jq -r '.[]|select(.code=="lp-u-veg")|.latest.id' "$BODY")" "$(cat "$(slot_id_file lp-u-veg)")" "слот lp-u-veg"
|
||||
runreq '{}'; need_code 202; RUN6="$(jb .id)"; wait_run "$RUN6"; need_eq "$RS" succeeded status
|
||||
ns="$(node_statuses "$RUN6")"
|
||||
need_eq "$ns" "00:skipped 01:skipped 02:skipped 03:skipped 10:skipped 20:skipped 21:skipped 30:skipped 31:skipped" узлы
|
||||
end "DELETE $c1; в слоте снова исходный растр; run ${RUN6:0:8} $RS, все узлы skipped"
|
||||
|
||||
begin L15 "Повторный полный прогон с замером ресурсов worker"
|
||||
cpu0=""
|
||||
if [ "$LOCAL_CHECKS" = 1 ] && [ -n "$WORKER_C" ]; then
|
||||
cpu0="$(docker exec "$WORKER_C" sh -c "awk '/usage_usec/{print \$2}' /sys/fs/cgroup/cpu.stat" 2>/dev/null || true)"
|
||||
fi
|
||||
runreq '{"from_stage":0}'; need_code 202; RUN7="$(jb .id)"; c1="$CODE"; wait_run "$RUN7"; need_eq "$RS" succeeded status
|
||||
ns="$(node_statuses "$RUN7")"
|
||||
need_eq "$ns" "00:succeeded 01:succeeded 02:succeeded 03:succeeded 10:succeeded 20:succeeded 21:succeeded 30:succeeded 31:succeeded" узлы
|
||||
res=""
|
||||
if [ -n "$cpu0" ]; then
|
||||
cpu1="$(docker exec "$WORKER_C" sh -c "awk '/usage_usec/{print \$2}' /sys/fs/cgroup/cpu.stat" 2>/dev/null || echo "$cpu0")"
|
||||
peak="$(docker exec "$WORKER_C" sh -c 'cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0')"
|
||||
res="; CPU worker $(awk -v a="$cpu0" -v b="$cpu1" 'BEGIN{printf "%.2f", (b-a)/1e6}') с (включая COG-конвертацию), memory.peak $(awk -v p="$peak" 'BEGIN{printf "%.2f", p/1e9}') ГБ (с учётом страничного кэша)"
|
||||
fi
|
||||
end "$c1; run ${RUN7:0:8} $RS за $("$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run-$RUN7.json")$res"
|
||||
|
||||
# ==================================================================== summary
|
||||
TOTAL=$((PASSN + FAILN + SKIPN))
|
||||
{
|
||||
echo "-----------------------------------------------------------------"
|
||||
echo "ИТОГО: $TOTAL проверок — PASS $PASSN, FAIL $FAILN, SKIP $SKIPN"
|
||||
echo "RESULT: $PASSN/$TOTAL PASS"
|
||||
echo "ЛП прогоны: RUN1=${RUN1:-} (from_stage 0), RUN4=${RUN4:-} (пользовательские параметры)"
|
||||
} | tee -a "$RESULTS/functional.txt"
|
||||
[ "$FAILN" -eq 0 ]
|
||||
Reference in New Issue
Block a user