Compare commits
3 Commits
main
...
stage8/sec
| Author | SHA1 | Date | |
|---|---|---|---|
| ca87d6961b | |||
| c6229c46f7 | |||
| 5b44cbda2f |
@@ -298,6 +298,7 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: "#/components/schemas/DatasetGroup" }
|
||||
"400": { $ref: "#/components/responses/BadRequest" }
|
||||
"413": { $ref: "#/components/responses/PayloadTooLarge" }
|
||||
"404": { $ref: "#/components/responses/NotFound" }
|
||||
"422": { $ref: "#/components/responses/ValidationError" }
|
||||
get:
|
||||
@@ -460,6 +461,7 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: "#/components/schemas/Dataset" }
|
||||
"400": { $ref: "#/components/responses/BadRequest" }
|
||||
"413": { $ref: "#/components/responses/PayloadTooLarge" }
|
||||
"422": { $ref: "#/components/responses/ValidationError" }
|
||||
|
||||
/datasets/{id}:
|
||||
@@ -677,6 +679,7 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: "#/components/schemas/Dataset" }
|
||||
"400": { $ref: "#/components/responses/BadRequest" }
|
||||
"413": { $ref: "#/components/responses/PayloadTooLarge" }
|
||||
"404": { $ref: "#/components/responses/NotFound" }
|
||||
"422": { $ref: "#/components/responses/ValidationError" }
|
||||
get:
|
||||
@@ -823,6 +826,7 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: "#/components/schemas/Dataset" }
|
||||
"400": { $ref: "#/components/responses/BadRequest" }
|
||||
"413": { $ref: "#/components/responses/PayloadTooLarge" }
|
||||
"404": { $ref: "#/components/responses/NotFound" }
|
||||
"422": { $ref: "#/components/responses/ValidationError" }
|
||||
|
||||
@@ -946,6 +950,11 @@ components:
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: "#/components/schemas/Error" }
|
||||
PayloadTooLarge:
|
||||
description: Request body exceeds the configured upload limit (HTTP_MAX_UPLOAD_BYTES, default 2 GiB)
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: "#/components/schemas/Error" }
|
||||
ValidationError:
|
||||
description: Input failed validation
|
||||
content:
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# Build context is the repo root: docker build -f build/package/Dockerfile .
|
||||
FROM golang:1.26-alpine3.24 AS builder
|
||||
# Go patch release pinned (not the floating 1.26 tag) so the shipped standard
|
||||
# library is reproducible and includes the security fixes up to go1.26.9;
|
||||
# keep in step with the `toolchain` line in go.mod.
|
||||
FROM golang:1.26.9-alpine3.24 AS builder
|
||||
|
||||
# godal (GDAL bindings) needs cgo, a C/C++ toolchain and the GDAL headers.
|
||||
# g++ is required: parts of the binding compile as C++ against GDAL's headers.
|
||||
@@ -14,7 +17,7 @@ COPY . .
|
||||
RUN CGO_ENABLED=1 GOOS=linux go build -trimpath -o /out/gis ./cmd/gis
|
||||
|
||||
# Must stay on the same Alpine release as the builder — hence the pinned
|
||||
# `golang:1.26-alpine3.24` above rather than the floating `-alpine` tag. A
|
||||
# `golang:1.26.9-alpine3.24` above rather than the floating `-alpine` tag. A
|
||||
# mismatch ships a different libgdal soname, and the image still builds green
|
||||
# while the binary dies at startup with "libgdal.so.NN: No such file".
|
||||
# Bump both lines together.
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
# HTTP server
|
||||
PORT=8080
|
||||
# Upper bound for any request body, uploads included (bytes; default 2 GiB).
|
||||
# Larger requests are rejected with 413. 0 disables the cap.
|
||||
HTTP_MAX_UPLOAD_BYTES=2147483648
|
||||
|
||||
# Postgres
|
||||
DB_URL=postgres://gis:gis@127.0.0.1:5432/gis?sslmode=disable
|
||||
|
||||
@@ -1,16 +1,21 @@
|
||||
# Credentials below default to well-known development values so that
|
||||
# `docker compose up` works out of the box. They MUST be overridden on any
|
||||
# shared or reachable host: put strong values for POSTGRES_PASSWORD,
|
||||
# MINIO_ROOT_USER, MINIO_ROOT_PASSWORD, RABBITMQ_DEFAULT_USER and
|
||||
# RABBITMQ_DEFAULT_PASS in deployments/.env (not committed).
|
||||
x-app-common: &app-common
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: build/package/Dockerfile
|
||||
environment: &app-environment
|
||||
PORT: 8080
|
||||
DB_URL: postgres://gis:gis@postgres:5432/gis?sslmode=disable
|
||||
DB_URL: postgres://gis:${POSTGRES_PASSWORD:-gis}@postgres:5432/gis?sslmode=disable
|
||||
S3_ENDPOINT: minio:9000
|
||||
S3_ACCESS_KEY: minioadmin
|
||||
S3_SECRET_KEY: minioadmin
|
||||
S3_ACCESS_KEY: ${MINIO_ROOT_USER:-minioadmin}
|
||||
S3_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-minioadmin}
|
||||
S3_BUCKET: geofiles
|
||||
S3_USE_SSL: "false"
|
||||
RABBITMQ_URL: amqp://guest:guest@rabbitmq:5672/
|
||||
RABBITMQ_URL: amqp://${RABBITMQ_DEFAULT_USER:-guest}:${RABBITMQ_DEFAULT_PASS:-guest}@rabbitmq:5672/
|
||||
RABBITMQ_EXCHANGE: gis.events
|
||||
RABBITMQ_QUEUE: gis.events.example
|
||||
depends_on:
|
||||
@@ -47,7 +52,7 @@ services:
|
||||
image: postgis/postgis:17-3.5
|
||||
environment:
|
||||
POSTGRES_USER: gis
|
||||
POSTGRES_PASSWORD: gis
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-gis}
|
||||
POSTGRES_DB: gis
|
||||
ports:
|
||||
- "${DB_PORT:-5432}:5432"
|
||||
@@ -67,8 +72,8 @@ services:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
@@ -84,8 +89,8 @@ services:
|
||||
- "5672:5672"
|
||||
- "15672:15672"
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: guest
|
||||
RABBITMQ_DEFAULT_PASS: guest
|
||||
RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER:-guest}
|
||||
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS:-guest}
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
|
||||
22
go.mod
22
go.mod
@@ -2,7 +2,7 @@ module gis
|
||||
|
||||
go 1.26.1
|
||||
|
||||
toolchain go1.26.2
|
||||
toolchain go1.26.9
|
||||
|
||||
require (
|
||||
github.com/airbusgeo/godal v0.0.18
|
||||
@@ -14,10 +14,10 @@ require (
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/minio/minio-go/v7 v7.2.0
|
||||
github.com/pressly/goose/v3 v3.27.1
|
||||
github.com/rabbitmq/amqp091-go v1.12.0
|
||||
github.com/rabbitmq/amqp091-go v1.13.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/text v0.37.0
|
||||
golang.org/x/sync v0.23.0
|
||||
golang.org/x/text v0.42.0
|
||||
modernc.org/sqlite v1.53.0
|
||||
)
|
||||
|
||||
@@ -31,10 +31,9 @@ require (
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/compress v1.18.7 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.21 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
@@ -46,13 +45,16 @@ require (
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/sethvargo/go-retry v0.3.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.9.0 // indirect
|
||||
github.com/stretchr/testify v1.12.1 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.52.0 // indirect
|
||||
golang.org/x/net v0.54.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/net v0.60.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
google.golang.org/protobuf v1.36.12 // indirect
|
||||
gopkg.in/ini.v1 v1.67.2 // indirect
|
||||
modernc.org/libc v1.73.4 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
|
||||
140
go.sum
140
go.sum
@@ -1,20 +1,58 @@
|
||||
cel.dev/expr v0.19.2 h1:V354PbqIXr9IQdwy4SYA4xa0HXaWq1BUPAGzugBY5V4=
|
||||
cel.dev/expr v0.19.2/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
|
||||
cloud.google.com/go v0.118.3 h1:jsypSnrE/w4mJysioGdMBg4MiW/hHx/sArFpaBWHdME=
|
||||
cloud.google.com/go v0.118.3/go.mod h1:Lhs3YLnBlwJ4KA6nuObNMZ/fCbOQBPuWKPoE0Wa/9Vc=
|
||||
cloud.google.com/go/auth v0.15.0 h1:Ly0u4aA5vG/fsSsxu98qCQBemXtAtJf+95z9HK+cxps=
|
||||
cloud.google.com/go/auth v0.15.0/go.mod h1:WJDGqZ1o9E9wKIL+IwStfyn/+s59zl4Bi+1KQNVXLZ8=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc=
|
||||
cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I=
|
||||
cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg=
|
||||
cloud.google.com/go/iam v1.4.1 h1:cFC25Nv+u5BkTR/BT1tXdoF2daiVbZ1RLx2eqfQ9RMM=
|
||||
cloud.google.com/go/iam v1.4.1/go.mod h1:2vUEJpUG3Q9p2UdsyksaKpDzlwOrnMzS30isdReIcLM=
|
||||
cloud.google.com/go/monitoring v1.24.0 h1:csSKiCJ+WVRgNkRzzz3BPoGjFhjPY23ZTcaenToJxMM=
|
||||
cloud.google.com/go/monitoring v1.24.0/go.mod h1:Bd1PRK5bmQBQNnuGwHBfUamAV1ys9049oEPHnn4pcsc=
|
||||
cloud.google.com/go/storage v1.51.0 h1:ZVZ11zCiD7b3k+cH5lQs/qcNaoSz3U9I0jgwVzqDlCw=
|
||||
cloud.google.com/go/storage v1.51.0/go.mod h1:YEJfu/Ki3i5oHC/7jyTgsGZwdQ8P9hqMqvpi5kRKGgc=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0=
|
||||
github.com/airbusgeo/errs v0.0.3 h1:+lUcbfm45tPLz/DMUC38QT+y9N60aKrPv4MXqVb7Noc=
|
||||
github.com/airbusgeo/errs v0.0.3/go.mod h1:/U1fafLfbdxZXtIqdTGeGGzLqnZcnGJXsJ0S7TaP4P0=
|
||||
github.com/airbusgeo/godal v0.0.18 h1:NQJ9P7OgSs5zVtlWH/tOta7QHYbbtha7Nttf63YM01Q=
|
||||
github.com/airbusgeo/godal v0.0.18/go.mod h1:Mpn/6exBzEbIx4DaG2rIjCmAiF9Ww4SJN2ZtwVsSwCo=
|
||||
github.com/airbusgeo/osio v0.1.3 h1:IxNf1co8lxwqJCziV+5qZyZbkT6JYdmR3L0oomNffrk=
|
||||
github.com/airbusgeo/osio v0.1.3/go.mod h1:d4qlaDSgsMilFoi+/62vwFZStuYIHIDovYzIRjTK+Jw=
|
||||
github.com/caarlos0/env/v11 v11.4.1 h1:fYwH0sWEsBSMPG7t4e/PEfTFzrWrpjyygXyUnWiSwEw=
|
||||
github.com/caarlos0/env/v11 v11.4.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk=
|
||||
github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-jose/go-jose/v4 v4.1.5 h1:RjgjO2LOtWOJKUC5wpwY9LR3B3vwVAz6JS2YHfYU6eA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.5/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
@@ -25,8 +63,16 @@ github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJ
|
||||
github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.5 h1:VgzTY2jogw3xt39CusEnFJWm7rlsq5yL5q9XdLOuP5g=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.5/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
|
||||
github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q=
|
||||
github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=
|
||||
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
|
||||
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
@@ -41,17 +87,13 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
|
||||
github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
|
||||
@@ -68,16 +110,15 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pressly/goose/v3 v3.27.1 h1:6uEvcprBybDmW4hcz3gYujhARhye+GoWKhEWyzD5sh4=
|
||||
github.com/pressly/goose/v3 v3.27.1/go.mod h1:maruOxsPnIG2yHHyo8UqKWXYKFcH7Q76csUV7+7KYoM=
|
||||
github.com/rabbitmq/amqp091-go v1.12.0 h1:V0v14Iqfs+MwHWihJt/nGS5Ulu0vw572b2Co3mwunkI=
|
||||
github.com/rabbitmq/amqp091-go v1.12.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rabbitmq/amqp091-go v1.13.0 h1:L8NA1WtF76C6KA3LAoufjfLgbist/If1UQYcsOjtxXA=
|
||||
github.com/rabbitmq/amqp091-go v1.13.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
@@ -88,6 +129,8 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spiffe/go-spiffe/v2 v2.9.0 h1:g5qTg3qIP1HXV+GzuyEfSJFVxUCpEP5Ui1OINVJHPB4=
|
||||
github.com/spiffe/go-spiffe/v2 v2.9.0/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
@@ -97,41 +140,74 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.34.0 h1:JRxssobiPg23otYU5SbWtQC//snGVIM3Tx6QRzlQBao=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.34.0/go.mod h1:cV4BMFcscUR/ckqLkbfQmF0PRsq8w/lMGzdbCSveBHo=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A=
|
||||
go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
||||
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
|
||||
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
|
||||
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
|
||||
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
||||
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
|
||||
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc=
|
||||
golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
|
||||
golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||
google.golang.org/api v0.224.0 h1:Ir4UPtDsNiwIOHdExr3fAj4xZ42QjK7uQte3lORLJwU=
|
||||
google.golang.org/api v0.224.0/go.mod h1:3V39my2xAGkodXy0vEqcEtkqgw2GtrFL5WuBZlCTCOQ=
|
||||
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE=
|
||||
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529 h1:XF8+t6QQiS0o9ArVan/HW8Q7cycNPGsJf6GA2nXxYAg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
|
||||
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss=
|
||||
gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c=
|
||||
modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
|
||||
@@ -108,15 +108,18 @@ func New(ctx context.Context) (*App, error) {
|
||||
// Handler builds the HTTP handler with all routes and readiness checks wired.
|
||||
func (a *App) Handler() stdhttp.Handler {
|
||||
validate := validator.New(validator.WithRequiredStructEnabled())
|
||||
validate.RegisterValidation("slug", func(fl validator.FieldLevel) bool {
|
||||
if err := validate.RegisterValidation("slug", func(fl validator.FieldLevel) bool {
|
||||
return domain.ValidSlug(fl.Field().String())
|
||||
})
|
||||
}); err != nil {
|
||||
// Only fails for an empty tag or nil func: a programming error.
|
||||
panic(fmt.Sprintf("register slug validation: %v", err))
|
||||
}
|
||||
|
||||
health := transporthttp.NewHealthHandler(map[string]transporthttp.ReadinessCheck{
|
||||
"postgres": func(ctx context.Context) error { return a.pool.Ping(ctx) },
|
||||
"s3": func(ctx context.Context) error { return a.store.Ping(ctx) },
|
||||
"rabbitmq": func(_ context.Context) error { return a.rabbit.Ping() },
|
||||
})
|
||||
}, a.Log)
|
||||
|
||||
return transporthttp.NewRouter(transporthttp.RouterDeps{
|
||||
Logger: a.Log,
|
||||
@@ -126,6 +129,7 @@ func (a *App) Handler() stdhttp.Handler {
|
||||
DatasetGroups: transporthttp.NewDatasetGroupHandler(a.groups, validate),
|
||||
Landscape: transporthttp.NewLandscapeHandler(a.landscape, validate),
|
||||
OpenAPISpec: api.Spec,
|
||||
MaxBodyBytes: a.Cfg.HTTP.MaxUploadBytes,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -27,6 +27,9 @@ type HTTPConfig struct {
|
||||
WriteTimeout time.Duration `env:"HTTP_WRITE_TIMEOUT" envDefault:"120s"`
|
||||
IdleTimeout time.Duration `env:"HTTP_IDLE_TIMEOUT" envDefault:"60s"`
|
||||
ShutdownTimeout time.Duration `env:"HTTP_SHUTDOWN_TIMEOUT" envDefault:"10s"`
|
||||
// MaxUploadBytes caps the size of any request body, multipart uploads
|
||||
// included (default 2 GiB); larger requests get 413. 0 disables the cap.
|
||||
MaxUploadBytes int64 `env:"HTTP_MAX_UPLOAD_BYTES" envDefault:"2147483648"`
|
||||
}
|
||||
|
||||
// Addr returns the listen address for the HTTP server.
|
||||
|
||||
@@ -156,12 +156,15 @@ func kernelMaxAnyValid(in [][]float64, out [][]int16, _ Params) {
|
||||
// S1..S5 (irrigation, snow, crop, pasture, tourism). Outputs: V1, V2, V3.
|
||||
func kernelFuncZoning(in [][]float64, out [][]int16, p Params) {
|
||||
e := in[0]
|
||||
// The int16 conversions below cannot overflow (gosec G115): thresholds and
|
||||
// tie_priority codes are limited to 1..5 by Params.Validate, and pixel
|
||||
// values are converted only after valid() has bounded them to 1..5.
|
||||
rules := [3]func(e, s int16) bool{
|
||||
func(e, s int16) bool {
|
||||
return (e <= 2 && s >= int16(p.V1MinSE12)) || (e == 3 && s >= int16(p.V1MinSE3))
|
||||
return (e <= 2 && s >= int16(p.V1MinSE12)) || (e == 3 && s >= int16(p.V1MinSE3)) // #nosec G115 -- 1..5, see above
|
||||
},
|
||||
func(e, s int16) bool {
|
||||
return e <= 2 || (e == 3 && s >= int16(p.V2MinSE3)) || (e == 4 && s >= int16(p.V2MinSE4))
|
||||
return e <= 2 || (e == 3 && s >= int16(p.V2MinSE3)) || (e == 4 && s >= int16(p.V2MinSE4)) // #nosec G115 -- 1..5, see above
|
||||
},
|
||||
func(e, s int16) bool { return e <= 4 },
|
||||
}
|
||||
@@ -205,7 +208,7 @@ func kernelFuncZoning(in [][]float64, out [][]int16, p Params) {
|
||||
for _, code := range p.TiePriority {
|
||||
k := code - 1
|
||||
if cand[k] && s[k] == maxS {
|
||||
sector = int16(code)
|
||||
sector = int16(code) // #nosec G115 -- tie_priority code is 1..5, see above
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,7 +112,9 @@ func gpkgScan(db *sql.DB, table string, names []string, limit int) []map[string]
|
||||
for i, n := range names {
|
||||
quoted[i] = quoteIdent(n)
|
||||
}
|
||||
query := fmt.Sprintf("SELECT %s FROM %s", strings.Join(quoted, ", "), quoteIdent(table))
|
||||
// Identifiers cannot be bound as parameters; every one is quoted with
|
||||
// quoteIdent (embedded quotes doubled) and the LIMIT is an int.
|
||||
query := fmt.Sprintf("SELECT %s FROM %s", strings.Join(quoted, ", "), quoteIdent(table)) // #nosec G201 -- identifiers quoted by quoteIdent
|
||||
if limit >= 0 {
|
||||
query += fmt.Sprintf(" LIMIT %d", limit)
|
||||
}
|
||||
|
||||
@@ -25,7 +25,8 @@ func NewGDALConverter() *GDALConverter {
|
||||
// ToCOG converts the source raster to a Cloud-Optimized GeoTIFF at dst. The COG
|
||||
// driver builds internal tiling and overviews.
|
||||
func (c *GDALConverter) ToCOG(ctx context.Context, src, dst string) error {
|
||||
cmd := exec.CommandContext(ctx, "gdal_translate",
|
||||
// Fixed binary, no shell; src/dst are service-generated temp-file paths.
|
||||
cmd := exec.CommandContext(ctx, "gdal_translate", // #nosec G204 -- see above
|
||||
"-of", "COG",
|
||||
"-co", "COMPRESS="+c.compression,
|
||||
src, dst,
|
||||
@@ -41,7 +42,7 @@ func (c *GDALConverter) ToCOG(ctx context.Context, src, dst string) error {
|
||||
// Footprint returns the raster's footprint as a GeoJSON polygon in EPSG:4326, or
|
||||
// nil if the raster has no spatial reference.
|
||||
func (c *GDALConverter) Footprint(ctx context.Context, src string) ([]byte, error) {
|
||||
out, err := exec.CommandContext(ctx, "gdalinfo", "-json", src).Output()
|
||||
out, err := exec.CommandContext(ctx, "gdalinfo", "-json", src).Output() // #nosec G204 -- fixed binary, no shell; src is a service temp file
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gdalinfo: %w", err)
|
||||
}
|
||||
@@ -72,7 +73,8 @@ func (c *GDALConverter) VectorFeatures(ctx context.Context, src string) ([]json.
|
||||
input = "/vsizip/" + src
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, "ogr2ogr",
|
||||
// Fixed binary, no shell; input is a service-generated temp-file path.
|
||||
cmd := exec.CommandContext(ctx, "ogr2ogr", // #nosec G204 -- see above
|
||||
"-f", "GeoJSON",
|
||||
"-t_srs", "EPSG:4326",
|
||||
"/vsistdout/", input,
|
||||
|
||||
@@ -692,7 +692,7 @@ func (s *DatasetService) downloadToTemp(ctx context.Context, key, pattern string
|
||||
|
||||
// uploadFile streams a local file to the object store.
|
||||
func (s *DatasetService) uploadFile(ctx context.Context, key, filePath, contentType string) error {
|
||||
f, err := os.Open(filePath)
|
||||
f, err := os.Open(filePath) // #nosec G304 -- filePath is a temp file this service created
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -728,7 +728,7 @@ func (s *LandscapeService) download(ctx context.Context, key, path string) error
|
||||
}
|
||||
defer obj.Close()
|
||||
|
||||
f, err := os.Create(path)
|
||||
f, err := os.Create(path) // #nosec G304 -- path is inside the node's os.MkdirTemp directory
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
63
internal/transport/http/content_disposition.go
Normal file
63
internal/transport/http/content_disposition.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// fallbackDownloadName is used when nothing printable survives sanitisation.
|
||||
const fallbackDownloadName = "download"
|
||||
|
||||
// contentDisposition builds a Content-Disposition header value for a stored,
|
||||
// user-supplied filename following RFC 6266: an ASCII-only quoted fallback in
|
||||
// filename= and the exact UTF-8 name, percent-encoded per RFC 8187, in
|
||||
// filename*=. Neither parameter can carry quotes, backslashes, control
|
||||
// characters (CR/LF) or parameter separators taken from the filename, so a
|
||||
// crafted upload name cannot inject extra header parameters or headers.
|
||||
func contentDisposition(disposition, filename string) string {
|
||||
name := strings.Map(func(r rune) rune {
|
||||
if r < 0x20 || r == 0x7f {
|
||||
return -1 // drop control characters, including CR and LF
|
||||
}
|
||||
return r
|
||||
}, filename)
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
name = fallbackDownloadName
|
||||
}
|
||||
|
||||
ascii := strings.Map(func(r rune) rune {
|
||||
if r > 0x7e || r == '"' || r == '\\' {
|
||||
return '_'
|
||||
}
|
||||
return r
|
||||
}, name)
|
||||
|
||||
return fmt.Sprintf(`%s; filename="%s"; filename*=UTF-8''%s`, disposition, ascii, encodeRFC8187(name))
|
||||
}
|
||||
|
||||
// encodeRFC8187 percent-encodes every byte of s that is not an attr-char
|
||||
// (RFC 8187 §3.2.1).
|
||||
func encodeRFC8187(s string) string {
|
||||
const hex = "0123456789ABCDEF"
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if isAttrChar(c) {
|
||||
b.WriteByte(c)
|
||||
continue
|
||||
}
|
||||
b.WriteByte('%')
|
||||
b.WriteByte(hex[c>>4])
|
||||
b.WriteByte(hex[c&0x0f])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func isAttrChar(c byte) bool {
|
||||
switch {
|
||||
case 'a' <= c && c <= 'z', 'A' <= c && c <= 'Z', '0' <= c && c <= '9':
|
||||
return true
|
||||
}
|
||||
return strings.IndexByte("!#$&+-.^_`|~", c) >= 0
|
||||
}
|
||||
57
internal/transport/http/content_disposition_test.go
Normal file
57
internal/transport/http/content_disposition_test.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"mime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestContentDisposition(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
want string
|
||||
}{
|
||||
{"plain", "roads.geojson", `attachment; filename="roads.geojson"; filename*=UTF-8''roads.geojson`},
|
||||
{"space", "my map.tif", `attachment; filename="my map.tif"; filename*=UTF-8''my%20map.tif`},
|
||||
{"quote injection", `a.tif"; filename="evil.exe`, `attachment; filename="a.tif_; filename=_evil.exe"; filename*=UTF-8''a.tif%22%3B%20filename%3D%22evil.exe`},
|
||||
{"crlf injection", "a.tif\r\nSet-Cookie: x=1", `attachment; filename="a.tifSet-Cookie: x=1"; filename*=UTF-8''a.tifSet-Cookie%3A%20x%3D1`},
|
||||
{"backslash", `..\..\a.sld`, `attachment; filename=".._.._a.sld"; filename*=UTF-8''..%5C..%5Ca.sld`},
|
||||
{"cyrillic", "карта.tif", `attachment; filename="_____.tif"; filename*=UTF-8''%D0%BA%D0%B0%D1%80%D1%82%D0%B0.tif`},
|
||||
{"empty", "", `attachment; filename="download"; filename*=UTF-8''download`},
|
||||
{"only controls", "\r\n\t", `attachment; filename="download"; filename*=UTF-8''download`},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := contentDisposition("attachment", tt.filename)
|
||||
if got != tt.want {
|
||||
t.Fatalf("contentDisposition(%q)\n got %s\nwant %s", tt.filename, got, tt.want)
|
||||
}
|
||||
if strings.ContainsAny(got, "\r\n") {
|
||||
t.Fatalf("header value contains CR/LF: %q", got)
|
||||
}
|
||||
// The value must parse as a single disposition whose decoded
|
||||
// filename is the sanitised original.
|
||||
disp, params, err := mime.ParseMediaType(got)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMediaType(%q): %v", got, err)
|
||||
}
|
||||
if disp != "attachment" {
|
||||
t.Fatalf("disposition = %q, want attachment", disp)
|
||||
}
|
||||
if len(params) != 1 {
|
||||
t.Fatalf("params = %v, want exactly one decoded filename", params)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentDispositionDecodesUTF8Name(t *testing.T) {
|
||||
_, params, err := mime.ParseMediaType(contentDisposition("inline", "карта 2026.cog.tif"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := params["filename"]; got != "карта 2026.cog.tif" {
|
||||
t.Fatalf("decoded filename = %q", got)
|
||||
}
|
||||
}
|
||||
@@ -177,8 +177,7 @@ func (h *DatasetGroupHandler) uploadSLD(w http.ResponseWriter, r *http.Request)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
|
||||
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
|
||||
if !parseMultipartForm(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -228,7 +227,7 @@ func (h *DatasetGroupHandler) downloadSLD(w http.ResponseWriter, r *http.Request
|
||||
filename = *group.SldFilename
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`)
|
||||
w.Header().Set("Content-Disposition", contentDisposition("attachment", filename))
|
||||
if _, err := io.Copy(w, obj); err != nil {
|
||||
// Headers are already sent; nothing useful to return to the client.
|
||||
return
|
||||
|
||||
@@ -18,9 +18,31 @@ import (
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// maxUploadBytes caps the in-memory portion of a multipart upload (64 MiB).
|
||||
// maxUploadBytes caps the in-memory portion of a multipart upload (64 MiB);
|
||||
// larger parts spill to temporary files. The total request size is capped by
|
||||
// the limitRequestBody middleware (HTTP_MAX_UPLOAD_BYTES).
|
||||
const maxUploadBytes = 64 << 20
|
||||
|
||||
// parseMultipartForm parses a multipart upload and writes the error response
|
||||
// itself when that fails: 413 when the body exceeds the router's size cap, 400
|
||||
// otherwise. It reports whether the handler may continue.
|
||||
func parseMultipartForm(w http.ResponseWriter, r *http.Request) bool {
|
||||
// #nosec G120 -- r.Body is wrapped in http.MaxBytesReader by the
|
||||
// limitRequestBody middleware that NewRouter installs on every route.
|
||||
err := r.ParseMultipartForm(maxUploadBytes)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
httputil.WriteError(w, http.StatusRequestEntityTooLarge,
|
||||
"request body exceeds the "+strconv.FormatInt(tooLarge.Limit, 10)+" byte limit")
|
||||
return false
|
||||
}
|
||||
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
|
||||
return false
|
||||
}
|
||||
|
||||
// DatasetHandler serves the /datasets routes.
|
||||
type DatasetHandler struct {
|
||||
svc *service.DatasetService
|
||||
@@ -121,8 +143,7 @@ func (h *DatasetHandler) mapping(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (h *DatasetHandler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
|
||||
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
|
||||
if !parseMultipartForm(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -583,7 +604,7 @@ func (h *DatasetHandler) geojson(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/geo+json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(fc)
|
||||
_, _ = w.Write(fc) // #nosec G705 -- JSON body served as application/geo+json with nosniff, never as HTML
|
||||
}
|
||||
|
||||
// cog streams the raster dataset's Cloud-Optimized GeoTIFF. It responds 422 for
|
||||
@@ -608,7 +629,7 @@ func (h *DatasetHandler) cog(w http.ResponseWriter, r *http.Request) {
|
||||
defer obj.Close()
|
||||
|
||||
w.Header().Set("Content-Type", "image/tiff")
|
||||
w.Header().Set("Content-Disposition", `inline; filename="`+cogFilename(dataset.Filename)+`"`)
|
||||
w.Header().Set("Content-Disposition", contentDisposition("inline", cogFilename(dataset.Filename)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := io.Copy(w, obj); err != nil {
|
||||
// Headers are already sent; nothing useful to return to the client.
|
||||
@@ -637,7 +658,7 @@ func (h *DatasetHandler) download(w http.ResponseWriter, r *http.Request) {
|
||||
defer obj.Close()
|
||||
|
||||
w.Header().Set("Content-Type", dataset.ContentType)
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="`+dataset.Filename+`"`)
|
||||
w.Header().Set("Content-Disposition", contentDisposition("attachment", dataset.Filename))
|
||||
if dataset.SizeBytes > 0 {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(dataset.SizeBytes, 10))
|
||||
}
|
||||
@@ -655,8 +676,7 @@ func (h *DatasetHandler) uploadSLD(w http.ResponseWriter, r *http.Request) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
|
||||
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
|
||||
if !parseMultipartForm(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -708,7 +728,7 @@ func (h *DatasetHandler) downloadSLD(w http.ResponseWriter, r *http.Request) {
|
||||
filename = *dataset.SldFilename
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`)
|
||||
w.Header().Set("Content-Disposition", contentDisposition("attachment", filename))
|
||||
if _, err := io.Copy(w, obj); err != nil {
|
||||
// Headers are already sent; nothing useful to return to the client.
|
||||
return
|
||||
|
||||
@@ -2,6 +2,7 @@ package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"gis/pkg/httputil"
|
||||
@@ -13,11 +14,17 @@ type ReadinessCheck func(ctx context.Context) error
|
||||
// HealthHandler serves liveness and readiness probes.
|
||||
type HealthHandler struct {
|
||||
checks map[string]ReadinessCheck
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewHealthHandler builds a HealthHandler with the given named readiness checks.
|
||||
func NewHealthHandler(checks map[string]ReadinessCheck) *HealthHandler {
|
||||
return &HealthHandler{checks: checks}
|
||||
// Failed checks are logged with their error; the response only names the
|
||||
// failing component, so connection details never reach the client.
|
||||
func NewHealthHandler(checks map[string]ReadinessCheck, log *slog.Logger) *HealthHandler {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
return &HealthHandler{checks: checks, log: log}
|
||||
}
|
||||
|
||||
// Live reports that the process is up.
|
||||
@@ -32,7 +39,8 @@ func (h *HealthHandler) Ready(w http.ResponseWriter, r *http.Request) {
|
||||
ready := true
|
||||
for name, check := range h.checks {
|
||||
if err := check(r.Context()); err != nil {
|
||||
results[name] = "error: " + err.Error()
|
||||
h.log.Warn("readiness check failed", "component", name, "error", err)
|
||||
results[name] = "unavailable"
|
||||
ready = false
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -58,8 +58,7 @@ func (h *LandscapeHandler) slots(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *LandscapeHandler) uploadSlot(w http.ResponseWriter, r *http.Request) {
|
||||
code := chi.URLParam(r, "code")
|
||||
|
||||
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
|
||||
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
|
||||
if !parseMultipartForm(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -31,3 +31,29 @@ func requestLogger(log *slog.Logger) func(http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// limitRequestBody caps every request body at maxBytes (http.MaxBytesReader),
|
||||
// so a client cannot stream an unbounded upload to memory or to the temporary
|
||||
// files multipart parsing spills to. A non-positive maxBytes disables the cap.
|
||||
// JSON endpoints apply their own, smaller cap in httputil.DecodeJSON.
|
||||
func limitRequestBody(maxBytes int64) func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
if maxBytes <= 0 {
|
||||
return next
|
||||
}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxBytes)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// securityHeaders sets response headers that are safe for every route.
|
||||
// nosniff stops browsers from MIME-sniffing user-supplied content (downloads,
|
||||
// GeoJSON) into an executable type.
|
||||
func securityHeaders(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
131
internal/transport/http/middleware_test.go
Normal file
131
internal/transport/http/middleware_test.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-playground/validator/v10"
|
||||
)
|
||||
|
||||
// multipartBody builds a multipart/form-data body with one file part of
|
||||
// size bytes.
|
||||
func multipartBody(t *testing.T, size int) (*bytes.Buffer, string) {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
mw := multipart.NewWriter(&buf)
|
||||
part, err := mw.CreateFormFile("file", "big.tif")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := part.Write(bytes.Repeat([]byte{'x'}, size)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &buf, mw.FormDataContentType()
|
||||
}
|
||||
|
||||
func TestLimitRequestBodyRejectsOversizedUpload(t *testing.T) {
|
||||
body, ct := multipartBody(t, 4096)
|
||||
req := httptest.NewRequest(http.MethodPost, "/landscape/slots/lp-z-lcr", body)
|
||||
req.Header.Set("Content-Type", ct)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
h := limitRequestBody(1024)(http.HandlerFunc(newLandscapeTestHandler().uploadSlot))
|
||||
h.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("status = %d, want %d (body %s)", rec.Code, http.StatusRequestEntityTooLarge, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "1024 byte limit") {
|
||||
t.Fatalf("body %s does not state the limit", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLimitRequestBodyPassesSmallBody(t *testing.T) {
|
||||
var got []byte
|
||||
h := limitRequestBody(1024)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var err error
|
||||
got, err = io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Errorf("read: %v", err)
|
||||
}
|
||||
}))
|
||||
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/", strings.NewReader("hello")))
|
||||
if string(got) != "hello" {
|
||||
t.Fatalf("body = %q, want hello", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLimitRequestBodyDisabled(t *testing.T) {
|
||||
h := limitRequestBody(0)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := io.ReadAll(r.Body); err != nil {
|
||||
t.Errorf("read with cap disabled: %v", err)
|
||||
}
|
||||
}))
|
||||
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(make([]byte, 1<<20))))
|
||||
}
|
||||
|
||||
func newTestRouter() http.Handler {
|
||||
v := validator.New(validator.WithRequiredStructEnabled())
|
||||
return NewRouter(RouterDeps{
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
Health: NewHealthHandler(nil, nil),
|
||||
Categories: NewCategoryHandler(nil, v),
|
||||
Datasets: NewDatasetHandler(nil, v),
|
||||
DatasetGroups: NewDatasetGroupHandler(nil, v),
|
||||
Landscape: NewLandscapeHandler(nil, v),
|
||||
MaxBodyBytes: 1024,
|
||||
})
|
||||
}
|
||||
|
||||
func TestRouterSetsNosniff(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
newTestRouter().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
||||
if got := rec.Header().Get("X-Content-Type-Options"); got != "nosniff" {
|
||||
t.Fatalf("X-Content-Type-Options = %q, want nosniff", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRouterCapsUploadBody(t *testing.T) {
|
||||
body, ct := multipartBody(t, 4096)
|
||||
req := httptest.NewRequest(http.MethodPost, "/landscape/slots/lp-z-lcr", body)
|
||||
req.Header.Set("Content-Type", ct)
|
||||
rec := httptest.NewRecorder()
|
||||
newTestRouter().ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("status = %d, want %d (body %s)", rec.Code, http.StatusRequestEntityTooLarge, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadyDoesNotLeakCheckErrors(t *testing.T) {
|
||||
h := NewHealthHandler(map[string]ReadinessCheck{
|
||||
"postgres": func(context.Context) error {
|
||||
return errors.New("dial tcp 10.1.2.3:5432: connect: connection refused (user=gis)")
|
||||
},
|
||||
"s3": func(context.Context) error { return nil },
|
||||
}, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.Ready(rec, httptest.NewRequest(http.MethodGet, "/readyz", nil))
|
||||
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusServiceUnavailable)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if strings.Contains(body, "10.1.2.3") || strings.Contains(body, "dial tcp") {
|
||||
t.Fatalf("readiness body leaks internal error details: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, `"postgres":"unavailable"`) || !strings.Contains(body, `"s3":"ok"`) {
|
||||
t.Fatalf("unexpected readiness body: %s", body)
|
||||
}
|
||||
}
|
||||
@@ -19,16 +19,23 @@ type RouterDeps struct {
|
||||
// Landscape serves the /landscape routes.
|
||||
Landscape *LandscapeHandler
|
||||
OpenAPISpec []byte
|
||||
// MaxBodyBytes caps every request body (uploads included); <= 0 disables
|
||||
// the cap.
|
||||
MaxBodyBytes int64
|
||||
}
|
||||
|
||||
// NewRouter builds the application's HTTP handler with middleware and routes.
|
||||
func NewRouter(deps RouterDeps) http.Handler {
|
||||
r := chi.NewRouter()
|
||||
|
||||
// middleware.RealIP is deliberately not used: it rewrites r.RemoteAddr
|
||||
// from client-controlled headers (X-Forwarded-For, X-Real-IP,
|
||||
// True-Client-IP) and is deprecated upstream as an IP-spoofing vector.
|
||||
r.Use(middleware.RequestID)
|
||||
r.Use(middleware.RealIP)
|
||||
r.Use(requestLogger(deps.Logger))
|
||||
r.Use(middleware.Recoverer)
|
||||
r.Use(securityHeaders)
|
||||
r.Use(limitRequestBody(deps.MaxBodyBytes))
|
||||
|
||||
// Health/readiness endpoints.
|
||||
r.Get("/healthz", deps.Health.Live)
|
||||
|
||||
2
test/acceptance/.gitignore
vendored
Normal file
2
test/acceptance/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
results/
|
||||
__pycache__/
|
||||
320
test/acceptance/README.md
Normal file
320
test/acceptance/README.md
Normal file
@@ -0,0 +1,320 @@
|
||||
# Пакет приёмочных испытаний (Этап 8)
|
||||
|
||||
Пакет позволяет Заказчику самостоятельно воспроизвести то, что заявлено в
|
||||
документах Этапа 8, на изолированном испытательном стенде, развёрнутом из
|
||||
исходного кода этой ветки:
|
||||
|
||||
| № | Что проверяется | Скрипт | Документ |
|
||||
|---|---|---|---|
|
||||
| 1 | 89 функциональных проверок испытательного стенда (F01–F38, N01–N38, L02–L05, L07–L15) | `functional.sh` | Приложение 7 «Протокол функционального тестирования», раздел 5 |
|
||||
| 2 | Попиксельная сверка расчёта Ландшафтной программы (ЛП) с независимой эталонной реализацией | `lp_gen.py`, `lp_run.sh`, `lp_verify.py` | ЛП техописание (правила пикселя, граф узлов); Приложение 7, L07/L08/L12 |
|
||||
| 3 | Нагрузочные испытания: 1000 одновременных клиентов, сценарии S1–S6 | `load.sh` (+ `seed.sh`) | Приложение 3 «Протокол нагрузочного тестирования» |
|
||||
| 4 | Повторный анализ безопасности: 0 уязвимостей High/Critical | `scan.sh` | Приложение 4 «Отчет об устранении уязвимостей» |
|
||||
|
||||
Все скрипты — bash (macOS и Linux, `set -euo pipefail`, совместимы с bash 3.2),
|
||||
curl, jq и Python 3. Результаты прогонов сохраняются в `test/acceptance/results/`
|
||||
(каталог не версионируется).
|
||||
|
||||
> **Внимание.** `functional.sh`, `lp_run.sh`, `seed.sh` и сценарий S5 из
|
||||
> `load.sh` **создают данные** (в том числе заполняют слоты ЛП). Их следует
|
||||
> запускать только на изолированном стенде `gis-acceptance`. На рабочем стенде
|
||||
> https://dssgis.dwh.kz допускается только `lp_verify.py` — он выполняет
|
||||
> исключительно GET-запросы (`lp_run.sh` и `seed.sh` отказываются работать с
|
||||
> этим адресом).
|
||||
|
||||
## Состав пакета
|
||||
|
||||
| Файл | Назначение |
|
||||
|---|---|
|
||||
| `compose.acceptance.yml` | Надстройка над `deployments/docker-compose.yml` для изолированного стенда (проект `gis-acceptance`) |
|
||||
| `stand.sh` | Развёртывание (`up`: сборка, миграции, справочник районов, ожидание `/readyz`) и удаление (`down`) стенда |
|
||||
| `functional.sh` | 89 функциональных проверок, построчный вывод `ID \| описание \| PASS/FAIL \| detail` |
|
||||
| `lp_gen.py` | Генерация 13 синтетических входных растров ЛП и одного невыровненного растра |
|
||||
| `lp_run.sh` | Загрузка входов в 13 слотов → прогон `{"from_stage":0}` → `lp_verify.py` |
|
||||
| `lp_verify.py` | Независимая (numpy) реализация всех узлов ЛП и попиксельная сверка 15 выходов; только GET |
|
||||
| `load.sh` | Сценарии нагрузки S1–S6 (hey 0.1.5), сводная таблица, CSV/JSON |
|
||||
| `seed.sh` | Наполнение стенда данными для нагрузочных испытаний (`seed.sh clean` — удаление) |
|
||||
| `scan.sh` | govulncheck (исходный код и бинарный файл), gosec, trivy fs, trivy image, golangci-lint |
|
||||
| `data/` | Тестовые векторные данные (участки, население и поголовье КРС по 14 районам СКО с реальными кодами КАТО), SLD-стили |
|
||||
| `lib/` | Вспомогательные скрипты: генерация NDVI, анализ вывода hey, Dockerfile генератора нагрузки, пересоздание `data/` |
|
||||
| `requirements.txt` | Python-зависимости (`numpy`, `rasterio`) |
|
||||
|
||||
## Требования
|
||||
|
||||
* Docker Engine с Docker Compose v2.24+ (используется тег `!override` для портов).
|
||||
* `curl`, `jq`, Python 3.10+ с `numpy` и `rasterio`:
|
||||
```sh
|
||||
python3 -m venv .venv && .venv/bin/pip install -r test/acceptance/requirements.txt
|
||||
export PYTHON=$PWD/.venv/bin/python # скрипты используют $PYTHON (по умолчанию python3)
|
||||
```
|
||||
* Для `scan.sh`: Go (версия toolchain берётся из `go.mod`), `govulncheck`,
|
||||
`gosec`, `trivy`; `golangci-lint` — по желанию.
|
||||
|
||||
## 0. Испытательный стенд
|
||||
|
||||
```sh
|
||||
git clone https://dssgit.dwh.kz/infureal/gis.git && cd gis && git checkout stage8/security-fixes
|
||||
test/acceptance/stand.sh up # сборка образа, запуск, migrate up, import-districts, ожидание /readyz
|
||||
```
|
||||
|
||||
`stand.sh up` выполняет те же шаги, что и вручную:
|
||||
|
||||
```sh
|
||||
C="docker compose -p gis-acceptance -f deployments/docker-compose.yml -f test/acceptance/compose.acceptance.yml"
|
||||
$C up -d --build --wait postgres minio rabbitmq
|
||||
$C run --rm --no-deps app migrate up # 11 миграций 00001…00011
|
||||
$C run --rm --no-deps app import-districts /stubs/districts.geojson # 14 районов СКО
|
||||
$C up -d app worker
|
||||
curl http://localhost:18080/readyz # {"components":{"postgres":"ok","rabbitmq":"ok","s3":"ok"},"ready":true}
|
||||
```
|
||||
|
||||
Порты стенда: API — 18080, PostgreSQL — 15432, MinIO — 19000 (API) / 19001
|
||||
(консоль), RabbitMQ — 15672 (только веб-интерфейс управления). Сеть контейнеров —
|
||||
`gis-acceptance_default`. Учётные данные — значения по умолчанию из
|
||||
`deployments/docker-compose.yml` (стенд изолированный).
|
||||
|
||||
Переменные окружения стенда:
|
||||
|
||||
| Переменная | По умолчанию | Назначение |
|
||||
|---|---|---|
|
||||
| `POSTGIS_PLATFORM` | `linux/amd64` | Образ `postgis/postgis:17-3.5` публикуется только для amd64. На Apple Silicon (arm64) он запускается в эмуляции, на amd64-хостах — нативно, поэтому значение по умолчанию работает на обеих платформах |
|
||||
| `MINIO_IMAGE` | `minio/minio:latest` | Если образ `minio/minio` недоступен в реестре, подойдёт любой образ с сервером `minio` и клиентом `mc`, например `MINIO_IMAGE=cgr.dev/chainguard/minio:latest` |
|
||||
| `APP_IMAGE` | `gis-acceptance-app:local` | Тег локально собранного образа приложения |
|
||||
| `HTTP_MAX_UPLOAD_BYTES` | `2147483648` | Предельный размер тела запроса (2 ГиБ) |
|
||||
| `LP_MAX_WORKERS` / `LP_MAX_NODES` | `8` / `2` | Параллелизм расчёта ЛП |
|
||||
|
||||
Удаление стенда вместе с томами: `test/acceptance/stand.sh down`
|
||||
(= `docker compose -p gis-acceptance down -v`).
|
||||
|
||||
## 1. Функциональные проверки — `functional.sh`
|
||||
|
||||
```sh
|
||||
test/acceptance/functional.sh # стенд http://localhost:18080
|
||||
BASE_URL=http://host:18080 LOCAL_CHECKS=0 test/acceptance/functional.sh # удалённый испытательный стенд
|
||||
```
|
||||
|
||||
| Переменная | По умолчанию | Назначение |
|
||||
|---|---|---|
|
||||
| `BASE_URL` | `http://localhost:18080` | Адрес API |
|
||||
| `LOCAL_CHECKS` | `auto` | `1` — подтверждение в PostGIS и MinIO через контейнеры проекта (`docker exec … psql`, `mc` в контейнере MinIO): F35 (часть MinIO), F36, F37, F38, ресурсы worker в L15. `0` — эти проверки пропускаются (`SKIP`). `auto` — `1`, если запущены контейнеры проекта `gis-acceptance` |
|
||||
| `COMPOSE_PROJECT` | `gis-acceptance` | Проект Compose для `LOCAL_CHECKS` |
|
||||
| `KEEP_DATA` | `0` | `1` — не удалять созданные данные |
|
||||
| `WAIT_TIMEOUT` | `300` | Ожидание асинхронной обработки, с |
|
||||
|
||||
Скрипт создаёт собственные тестовые данные (все коды имеют уникальный префикс
|
||||
`acc<время>-`), никогда не изменяет и не удаляет существовавшие объекты и в
|
||||
конце удаляет всё созданное: наборы данных (в том числе входы и выходы ЛП своих
|
||||
прогонов), группы, категории. Записи прогонов ЛП остаются — API не
|
||||
предусматривает их удаления. Порядок и смысл проверок совпадают с таблицами
|
||||
раздела 5 Приложения 7.
|
||||
|
||||
Ожидаемый вывод (фрагмент):
|
||||
|
||||
```
|
||||
F01 | Создание корневой категории POST /categories | PASS | 201; acc1792084680-ecology «Экология», parent_id null
|
||||
...
|
||||
N38 | Загрузка 100 МБ мусора с сигнатурой TIFF | PASS | 201 (104 857 604 байт за 0.5 с) → status failed, parse_error от gdal_translate; /readyz 200, сервис доступен
|
||||
...
|
||||
L07 | Попиксельная сверка 15 выходов с эталонной реализацией | PASS | 15 выходов × 6 000 000 пикселей; «TOTAL mismatching pixels: 0 -> PASS» …
|
||||
...
|
||||
ИТОГО: 89 проверок — PASS 89, FAIL 0, SKIP 0
|
||||
RESULT: 89/89 PASS
|
||||
```
|
||||
|
||||
Код возврата ненулевой при любом `FAIL`. Журнал и вспомогательные выводы
|
||||
(`L07_lp_verify.txt`, `L08_points.txt`, `L12_lp_verify.txt`, `F36.txt`) —
|
||||
в `results/functional-<время>/`.
|
||||
|
||||
### Соответствие Приложению 7 и уточнения методики
|
||||
|
||||
| Проверки | Раздел Приложения 7 | Как проверяется в пакете |
|
||||
|---|---|---|
|
||||
| F01–F09 | 5.2 Категории | HTTP-код и поля ответа, контроль цикла, дубликат, удаление |
|
||||
| F10–F34, N01–N13 | 5.3 Импорт/экспорт, SLD, группы, фильтры | Загрузка `data/plots_nko.geojson` (vector), `data/population_kato.geojson` и `data/livestock_kato.gpkg` (vector_with_kato), растра NDVI 600×400 EPSG:4326 Float32 (генерируется `lib/helpers.py`) |
|
||||
| N14–N38, F35 | 5.4 Негативные проверки | 12 некорректных загрузок, 10 эндпоинтов с несуществующим id, 4 — с некорректным id, некорректные тела JSON, параметры ЛП, 100 МБ мусора |
|
||||
| F36–F38 | 5.5 PostGIS и MinIO | SQL `ST_GeometryType/ST_SRID/ST_IsValid`, соединение наблюдений с `districts`, `mc stat` объектов |
|
||||
| L02–L15 | 5.6 Ландшафтная программа | 13 растров `lp_gen.py`, прогоны, `lp_verify.py`, инкрементальный пересчёт, `from_stage`, параметры, невыровненная сетка |
|
||||
|
||||
Уточнения относительно протокола (смысл и ожидаемые результаты проверок не
|
||||
изменены):
|
||||
|
||||
* **Коды объектов** имеют префикс прогона (`acc…-ecology` вместо `ecology`),
|
||||
чтобы повторный запуск не конфликтовал с данными прошлых прогонов.
|
||||
* **F33, F34** — выборки ограничены собственной категорией (`category_id`), чтобы
|
||||
результат не зависел от других данных стенда; во втором фильтре F34
|
||||
(`file_type + status`) добавлен `category_id`.
|
||||
* **F12** — охват проверяется по координатам выгруженного GeoJSON (вместо
|
||||
`ogrinfo`); **F22, L05** — формат COG проверяется через rasterio/GDAL
|
||||
(`LAYOUT=COG`, сжатие, размер, EPSG, тип, NoData) вместо `gdalinfo`;
|
||||
**L08** — 9 точек проверяются `lp_verify.py --points` вместо `gdallocationinfo`.
|
||||
* **F38** — проверяется наличие всех 7 объектов, созданных скриптом (4 исходных
|
||||
файла, COG, SLD набора, SLD группы), а не общее число объектов в бакете.
|
||||
* **F16, F17** — эталонные значения берутся из `data/population_kato.geojson`
|
||||
(значения, приведённые в протоколе, в файле сохранены).
|
||||
* **L05** — перед чтением `/landscape/results` скрипт дожидается статуса `ready`
|
||||
всех 15 выходов: конвертация выходов в COG выполняется worker'ом асинхронно
|
||||
после завершения прогона, а `/landscape/results` возвращает только готовые слои.
|
||||
* **N38 и ограничение размера тела запроса.** В этой ветке (исправления по
|
||||
Приложению 4) тело запроса ограничено `HTTP_MAX_UPLOAD_BYTES` (по умолчанию
|
||||
2 ГиБ); загрузка сверх предела отклоняется с **413** `request body exceeds
|
||||
the … byte limit`. Файл N38 (100 МБ) меньше предела, поэтому ожидание N38
|
||||
не изменилось: 201, затем `status = failed`. Проверить 413 можно вручную,
|
||||
запустив стенд с `HTTP_MAX_UPLOAD_BYTES=1048576` и загрузив файл больше 1 МиБ.
|
||||
* **`/readyz`** в этой ветке не раскрывает текст ошибок зависимостей: при сбое
|
||||
компонент помечается `"unavailable"` (ранее — `"error: <текст>"`). На исправном
|
||||
стенде ответ не изменился: `200 {"components":{"postgres":"ok","rabbitmq":"ok","s3":"ok"},"ready":true}`;
|
||||
N38 проверяет `200` и `ready = true`.
|
||||
* **L15** — PASS определяется успешным полным прогоном (202, `succeeded`, 9 узлов
|
||||
`succeeded`); время CPU и `memory.peak` worker (cgroup v2) выводятся
|
||||
справочно и включают COG-конвертацию выходов.
|
||||
|
||||
## 2. Ландшафтная программа: попиксельная сверка
|
||||
|
||||
### Входные данные — `lp_gen.py`
|
||||
|
||||
```sh
|
||||
$PYTHON test/acceptance/lp_gen.py --out /tmp/lp-in [--width 3000 --height 2000]
|
||||
```
|
||||
|
||||
13 растров `<код слота>.tif`: Int16, EPSG:32642, пиксель 100 м, 3000×2000,
|
||||
классы 1…5 из гладких тригонометрических полей (у каждого слота своё),
|
||||
рамка NoData −9999 шириной 40 пикселей. Особые случаи для правил пикселя:
|
||||
в `lp-u-crop` — блок NoData 200×200 (U считается по water, soil и veg),
|
||||
в `lp-u-veg` — блок NoData 100×100, наполовину перекрывающий его (там нет ни
|
||||
crop, ни veg → U = NoData), в `lp-p-snow` — блоки со значениями 0 и 6
|
||||
(не NoData, но вне 1…5 → недопустимы). Плюс `misaligned-lp-u-veg.tif`
|
||||
(1500×1000, 200 м) для проверки L13. Генерация детерминирована.
|
||||
|
||||
### Прогон на стенде — `lp_run.sh`, режим (a)
|
||||
|
||||
```sh
|
||||
test/acceptance/lp_run.sh # генерация → 13 слотов → {"from_stage":0} → lp_verify.py
|
||||
```
|
||||
|
||||
Входы и выходы остаются на стенде, прогон можно перепроверить:
|
||||
`$PYTHON test/acceptance/lp_verify.py --base-url http://localhost:18080 --run <run-id>`.
|
||||
|
||||
### Проверка любого стенда (только чтение) — `lp_verify.py`, режим (b)
|
||||
|
||||
```sh
|
||||
$PYTHON test/acceptance/lp_verify.py --base-url https://dssgis.dwh.kz --run <run-id> [--cache DIR]
|
||||
```
|
||||
|
||||
Скрипт выполняет только GET-запросы:
|
||||
|
||||
1. `GET /landscape/runs/{id}` — параметры прогона, для каждого узла `inputs` и
|
||||
`outputs` (группа → набор данных);
|
||||
2. `GET /datasets/{id}.cog` — 13 входов (наборы слотов, которые прогон
|
||||
фактически использовал) и 15 выходов (`--inputs-from download` читает
|
||||
входы через `/datasets/{id}/download` — исходный файл, который читал узел).
|
||||
|
||||
Ожидаемые значения рассчитываются по всей цепочке **только из 13 входов**
|
||||
(E — из пересчитанных Z и U, C и V — из пересчитанного E), поэтому ошибка в
|
||||
промежуточном слое не может скрыть ошибку в последующем. Дополнительно
|
||||
проверяется согласованность цепочки самого прогона (узел 10 читал выход узла
|
||||
00 этого прогона и т. д.) и совпадение сетки всех 28 растров. Растры
|
||||
обрабатываются окнами 512×4096 пикселей (выровнены по тайлам COG 512), поэтому
|
||||
расход памяти ограничен (~0,5 ГБ) и для растров рабочего стенда 22219×14977.
|
||||
|
||||
Правила пикселя (источник истины — `internal/landscape/kernels.go`, они же —
|
||||
в ЛП техописании): значение допустимо при `1 ≤ v ≤ 5`; `classify` в float32
|
||||
(`≤1.8→1, ≤2.6→2, ≤3.4→3, ≤4.2→4, ≤5.0→5`); z_integrated — среднее четырёх Z с
|
||||
весом 0,25; u_integrated — water ∧ soil ∧ (crop ∨ veg), веса 0,25 только у
|
||||
допустимых crop/veg; z_max/u_max — максимум; E = max(Z, U); C_j = min(P_j, E);
|
||||
p_max/c_max — максимум по допустимым; func_zoning V1–V3 с порогами
|
||||
`v1_min_s_e12`, `v1_min_s_e3`, `v2_min_s_e3`, `v2_min_s_e4` и `tie_priority`
|
||||
из параметров прогона.
|
||||
|
||||
Вывод: по каждому из 15 выходов — число пикселей, допустимых (по эталону) и
|
||||
**число несовпадающих пикселей** (с координатами первого расхождения), итоговая
|
||||
строка `TOTAL mismatching pixels: N -> PASS/FAIL`. Код возврата: 0 — полное
|
||||
совпадение, 1 — есть расхождения, 2 — ошибка данных (прогон не `succeeded`,
|
||||
не найден вход/выход, разные сетки).
|
||||
|
||||
## 3. Нагрузочные испытания — `load.sh`
|
||||
|
||||
```sh
|
||||
test/acceptance/load.sh # S1–S6, генератор внутри сети стенда
|
||||
SCENARIOS="S1 S3 S6" test/acceptance/load.sh # часть сценариев
|
||||
GEN=host TARGET_URL=http://localhost:18080 test/acceptance/load.sh # hey на хосте, через порт стенда
|
||||
NETWORK=bridge TARGET_URL=https://host BASE_URL=https://host SCENARIOS="S1 S3" test/acceptance/load.sh
|
||||
```
|
||||
|
||||
Генератор — hey 0.1.5, собирается из исходного кода в образ
|
||||
`gis-acceptance-hey:0.1.5` (`lib/hey.Dockerfile`, на тех же закреплённых образах
|
||||
Go/Alpine, что и приложение) и по умолчанию запускается **внутри сети стенда**
|
||||
(`docker run --network gis-acceptance_default …`, цель `http://app:8080`).
|
||||
Перед нагрузкой `seed.sh` (идемпотентно) наполняет стенд: 25 vector, 25
|
||||
vector_with_kato (с маппингом КАТО, 14 районов × 3 года) и 50 raster в
|
||||
категории `acc-load`; если прогонов ЛП ещё нет, выполняется `lp_run.sh`.
|
||||
На удалённом стенде без `ALLOW_WRITES=1` данные не создаются: id для
|
||||
эндпоинтов определяются GET-запросами, сценарий S5 пропускается.
|
||||
|
||||
Эндпоинты (табл. 5 Приложения 3): `/datasets?page_size=20`, `/datasets/{id}`,
|
||||
`/datasets/{id}.geojson`, `/datasets/{id}/observations`, `/categories`,
|
||||
`/dataset-groups/{id}/latest`, `/landscape/slots`, `/landscape/results`,
|
||||
`/landscape/runs/{id}`, `/readyz`.
|
||||
|
||||
| Сценарий | Параметры hey |
|
||||
|---|---|
|
||||
| S1 | `-n 100 -c 1` последовательно по 10 эндпоинтам (1000 запросов) |
|
||||
| S2 | 1000 запросов, 100 клиентов: 10 параллельных процессов `-n 100 -c 10` |
|
||||
| S3.1 | `-n 1000 -c 1000 /datasets?page_size=20`, 3 прогона |
|
||||
| S3.2 | `-n 1000 -c 1000 /landscape/results`, 3 прогона |
|
||||
| S4 | `-z 60s -q 1`, 1000 клиентов в 8 процессах: 300 `/datasets?page_size=20`, 150 `/datasets/{id}`, по 100 `.geojson`, `/observations`, `/categories`, `/latest`, `/landscape/results`, 50 `/readyz` |
|
||||
| S5 | `-z 30s -c 10` × 10 эндпоинтов + 20 параллельных `POST /datasets` (10 GeoJSON + 10 GeoTIFF), все загрузки должны дойти до `ready` (затем удаляются) |
|
||||
| S6 | Повтор S1 и `GET /readyz` после нагрузки |
|
||||
|
||||
S3.3 протокола (1000 одновременных запросов через опубликованный порт)
|
||||
воспроизводится запуском `GEN=host TARGET_URL=http://localhost:18080 SCENARIOS=S3`.
|
||||
|
||||
Вывод: сводная таблица (запросы, клиенты, длительность, RPS, доля успешных
|
||||
ответов, min/avg/P50/P90/P95/P99/max в мс, коды ответов, транспортные ошибки,
|
||||
итог по критериям), таблицы по эндпоинтам/прогонам для S2–S5, результаты
|
||||
загрузок S5, пиковое потребление CPU/памяти контейнерами (docker stats во время
|
||||
S4/S5) и критерий K4. Как и в протоколе, перцентили сценариев из нескольких
|
||||
процессов/прогонов — средневзвешенные по числу ответов. Файлы:
|
||||
`results/load-<время>/summary.csv`, `summary.json`, `detail.csv`, `raw/*.txt`
|
||||
(исходные выводы hey). Критерии (табл. 8 Приложения 3): K1 — успешных ответов
|
||||
≥ 99 %, K2 — ответов 5xx нет, K3 — P95 ≤ 2000 мс, K4 — после нагрузки `/readyz`
|
||||
200 и P50 S6 не хуже P50 S1 (допуск ×2 + 1 мс). Код возврата 1 при
|
||||
невыполнении любого критерия.
|
||||
|
||||
Генератор и стенд работают на одной машине и делят CPU, поэтому абсолютные
|
||||
значения RPS зависят от оборудования (протокол фиксирует их как нижнюю оценку).
|
||||
|
||||
## 4. Анализ безопасности — `scan.sh`
|
||||
|
||||
```sh
|
||||
test/acceptance/scan.sh
|
||||
```
|
||||
|
||||
| Инструмент | Что проверяет | Что считается High/Critical |
|
||||
|---|---|---|
|
||||
| `govulncheck ./...` | Уязвимости Go-модулей и стандартной библиотеки, достижимые из кода | Каждая уязвимость с цепочкой вызовов из кода (у govulncheck нет шкалы severity) |
|
||||
| `govulncheck -mode=binary` | Бинарный файл `/usr/local/bin/gis` из собранного образа | Каждая уязвимость с затронутыми символами в бинарнике |
|
||||
| `gosec ./...` | Статический анализ кода | Severity `HIGH` |
|
||||
| `trivy fs` | Зависимости (`go.mod`), секреты, конфигурации (Dockerfile, compose) | `HIGH`, `CRITICAL` |
|
||||
| `trivy image` | Образ приложения (Alpine + GDAL + бинарник), собирается из `build/package/Dockerfile` | `HIGH`, `CRITICAL` |
|
||||
| `golangci-lint` | Качество кода (если установлен) | Справочно: число замечаний (шкалы severity нет) |
|
||||
|
||||
Скрипт печатает таблицу «инструмент | цель | High | Critical | примечания» и
|
||||
итог `RESULT: 0 High / 0 Critical -> PASS`; код возврата 1 при наличии
|
||||
High/Critical. Полные отчёты (JSON) — `results/scan-<время>/`. Без Docker:
|
||||
`SKIP_IMAGE=1` (бинарник собирается локально с cgo и GDAL, `trivy image`
|
||||
пропускается).
|
||||
|
||||
## Порядок полного воспроизведения
|
||||
|
||||
```sh
|
||||
test/acceptance/stand.sh up
|
||||
test/acceptance/functional.sh | tee functional.txt # RESULT: 89/89 PASS
|
||||
test/acceptance/lp_run.sh | tee lp_local.txt # TOTAL mismatching pixels: 0 -> PASS
|
||||
$PYTHON test/acceptance/lp_verify.py --base-url https://dssgis.dwh.kz --run <run-id> | tee lp_prod.txt
|
||||
test/acceptance/load.sh | tee load.txt # критерии K1–K4 выполнены
|
||||
test/acceptance/scan.sh | tee scan.txt # RESULT: 0 High / 0 Critical -> PASS
|
||||
test/acceptance/stand.sh down
|
||||
```
|
||||
|
||||
Тестовые векторные данные `data/` пересоздаются командой
|
||||
`python3 test/acceptance/lib/make_fixtures.py` (нужен `ogr2ogr`).
|
||||
55
test/acceptance/compose.acceptance.yml
Normal file
55
test/acceptance/compose.acceptance.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
# Isolated acceptance-test stand (Приложения 3, 4, 7 к отчёту Этапа 8).
|
||||
#
|
||||
# Overlay on deployments/docker-compose.yml. Always use the project name
|
||||
# gis-acceptance so the stand never collides with a development or production
|
||||
# stack on the same host:
|
||||
#
|
||||
# docker compose -p gis-acceptance \
|
||||
# -f deployments/docker-compose.yml \
|
||||
# -f test/acceptance/compose.acceptance.yml up -d --build
|
||||
#
|
||||
# Host ports: app 18080, postgres 15432, minio 19000 (API) / 19001 (console),
|
||||
# rabbitmq 15672 (management UI only; AMQP stays inside the docker network).
|
||||
#
|
||||
# Environment knobs (all optional):
|
||||
# POSTGIS_PLATFORM platform of postgis/postgis (default linux/amd64: the image
|
||||
# has no arm64 build; Apple Silicon runs it under emulation,
|
||||
# amd64 hosts run it natively, so the default works on both)
|
||||
# MINIO_IMAGE MinIO image (default minio/minio:latest). Any image that
|
||||
# ships the `minio` server and the `mc` client works, e.g.
|
||||
# cgr.dev/chainguard/minio:latest.
|
||||
# APP_IMAGE tag for the locally built application image
|
||||
# HTTP_MAX_UPLOAD_BYTES, LP_MAX_WORKERS, LP_MAX_NODES — passed to app/worker.
|
||||
name: gis-acceptance
|
||||
|
||||
services:
|
||||
app:
|
||||
image: ${APP_IMAGE:-gis-acceptance-app:local}
|
||||
environment:
|
||||
HTTP_MAX_UPLOAD_BYTES: ${HTTP_MAX_UPLOAD_BYTES:-2147483648}
|
||||
LP_MAX_WORKERS: ${LP_MAX_WORKERS:-8}
|
||||
LP_MAX_NODES: ${LP_MAX_NODES:-2}
|
||||
ports: !override
|
||||
- "18080:8080"
|
||||
|
||||
worker:
|
||||
image: ${APP_IMAGE:-gis-acceptance-app:local}
|
||||
environment:
|
||||
HTTP_MAX_UPLOAD_BYTES: ${HTTP_MAX_UPLOAD_BYTES:-2147483648}
|
||||
LP_MAX_WORKERS: ${LP_MAX_WORKERS:-8}
|
||||
LP_MAX_NODES: ${LP_MAX_NODES:-2}
|
||||
|
||||
postgres:
|
||||
platform: ${POSTGIS_PLATFORM:-linux/amd64}
|
||||
ports: !override
|
||||
- "15432:5432"
|
||||
|
||||
minio:
|
||||
image: ${MINIO_IMAGE:-minio/minio:latest}
|
||||
ports: !override
|
||||
- "19000:9000"
|
||||
- "19001:9001"
|
||||
|
||||
rabbitmq:
|
||||
ports: !override
|
||||
- "15672:15672"
|
||||
19
test/acceptance/data/group_style.sld
Normal file
19
test/acceptance/data/group_style.sld
Normal file
@@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<sld:StyledLayerDescriptor version="1.0.0" xmlns:sld="http://www.opengis.net/sld" xmlns:ogc="http://www.opengis.net/ogc">
|
||||
<sld:NamedLayer>
|
||||
<sld:Name>group</sld:Name>
|
||||
<sld:UserStyle>
|
||||
<sld:Title>Group fallback style</sld:Title>
|
||||
<sld:FeatureTypeStyle>
|
||||
<sld:Rule>
|
||||
<sld:RasterSymbolizer>
|
||||
<sld:ColorMap>
|
||||
<sld:ColorMapEntry color="#f7fbff" quantity="0"/>
|
||||
<sld:ColorMapEntry color="#08306b" quantity="100000"/>
|
||||
</sld:ColorMap>
|
||||
</sld:RasterSymbolizer>
|
||||
</sld:Rule>
|
||||
</sld:FeatureTypeStyle>
|
||||
</sld:UserStyle>
|
||||
</sld:NamedLayer>
|
||||
</sld:StyledLayerDescriptor>
|
||||
BIN
test/acceptance/data/livestock_kato.gpkg
Normal file
BIN
test/acceptance/data/livestock_kato.gpkg
Normal file
Binary file not shown.
145
test/acceptance/data/plots_nko.geojson
Normal file
145
test/acceptance/data/plots_nko.geojson
Normal file
@@ -0,0 +1,145 @@
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"plot_id": 1,
|
||||
"name": "Участок 1",
|
||||
"area_ha": 1680.0
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[
|
||||
69.0,
|
||||
53.6
|
||||
],
|
||||
[
|
||||
69.6,
|
||||
53.6
|
||||
],
|
||||
[
|
||||
69.6,
|
||||
54.0
|
||||
],
|
||||
[
|
||||
69.0,
|
||||
54.0
|
||||
],
|
||||
[
|
||||
69.0,
|
||||
53.6
|
||||
]
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"plot_id": 2,
|
||||
"name": "Участок 2",
|
||||
"area_ha": 2100.0
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[
|
||||
69.9,
|
||||
54.2
|
||||
],
|
||||
[
|
||||
70.5,
|
||||
54.2
|
||||
],
|
||||
[
|
||||
70.5,
|
||||
54.7
|
||||
],
|
||||
[
|
||||
69.9,
|
||||
54.7
|
||||
],
|
||||
[
|
||||
69.9,
|
||||
54.2
|
||||
]
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"plot_id": 3,
|
||||
"name": "Участок 3",
|
||||
"area_ha": 1400.0
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[
|
||||
70.6,
|
||||
53.7
|
||||
],
|
||||
[
|
||||
71.1,
|
||||
53.7
|
||||
],
|
||||
[
|
||||
71.1,
|
||||
54.1
|
||||
],
|
||||
[
|
||||
70.6,
|
||||
54.1
|
||||
],
|
||||
[
|
||||
70.6,
|
||||
53.7
|
||||
]
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"plot_id": 4,
|
||||
"name": "Участок 4",
|
||||
"area_ha": 1750.0
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[
|
||||
71.2,
|
||||
54.5
|
||||
],
|
||||
[
|
||||
71.7,
|
||||
54.5
|
||||
],
|
||||
[
|
||||
71.7,
|
||||
55.0
|
||||
],
|
||||
[
|
||||
71.2,
|
||||
55.0
|
||||
],
|
||||
[
|
||||
71.2,
|
||||
54.5
|
||||
]
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
243
test/acceptance/data/population_kato.geojson
Normal file
243
test/acceptance/data/population_kato.geojson
Normal file
@@ -0,0 +1,243 @@
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "591000000",
|
||||
"district": "Петропавловск Г.А.",
|
||||
"Y2022": 47567.6,
|
||||
"Y2023": 48311.2,
|
||||
"Y2024": 49120.5
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
69.14998,
|
||||
54.89148
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "593200000",
|
||||
"district": "Айыртауский район",
|
||||
"Y2022": 21802.7,
|
||||
"Y2023": 22615.1,
|
||||
"Y2024": 23427.5
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
68.01276,
|
||||
53.19148
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "593400000",
|
||||
"district": "Акжарский район",
|
||||
"Y2022": 23605.4,
|
||||
"Y2023": 24417.8,
|
||||
"Y2024": 25230.2
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
71.74765,
|
||||
53.78951
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "595800000",
|
||||
"district": "Аккайынский район",
|
||||
"Y2022": 25264.0,
|
||||
"Y2023": 26076.4,
|
||||
"Y2024": 26888.8
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
69.55798,
|
||||
54.45755
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "594200000",
|
||||
"district": "Есильский район",
|
||||
"Y2022": 27066.7,
|
||||
"Y2023": 27879.1,
|
||||
"Y2024": 28691.5
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
68.32766,
|
||||
54.11517
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "594600000",
|
||||
"district": "Жамбылский район",
|
||||
"Y2022": 28869.4,
|
||||
"Y2023": 29681.8,
|
||||
"Y2024": 30494.2
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
66.98273,
|
||||
54.4391
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "595000000",
|
||||
"district": "Кызылжарский район",
|
||||
"Y2022": 46862.7,
|
||||
"Y2023": 48268.6,
|
||||
"Y2024": 49205.8
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
69.25907,
|
||||
54.89353
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "595200000",
|
||||
"district": "Мамлютский район",
|
||||
"Y2022": 32330.7,
|
||||
"Y2023": 33143.1,
|
||||
"Y2024": 33955.5
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
68.21021,
|
||||
54.81946
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "596000000",
|
||||
"district": "Тайыншинский район",
|
||||
"Y2022": 34133.4,
|
||||
"Y2023": 34945.8,
|
||||
"Y2024": 35758.2
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
69.89444,
|
||||
53.80885
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "596200000",
|
||||
"district": "Тимирязевский район",
|
||||
"Y2022": 35792.0,
|
||||
"Y2023": 36604.4,
|
||||
"Y2024": 37416.8
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
66.56047,
|
||||
53.66856
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "596400000",
|
||||
"district": "Уалихановский район",
|
||||
"Y2022": 37594.7,
|
||||
"Y2023": 38407.1,
|
||||
"Y2024": 39219.5
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
72.82078,
|
||||
53.36512
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "593600000",
|
||||
"district": "Район Магжана Жумабаева",
|
||||
"Y2022": 39253.3,
|
||||
"Y2023": 40065.7,
|
||||
"Y2024": 40878.1
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
70.5439,
|
||||
54.63954
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "595600000",
|
||||
"district": "Район Шал акына",
|
||||
"Y2022": 41056.0,
|
||||
"Y2023": 41868.4,
|
||||
"Y2024": 42680.8
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
67.40455,
|
||||
53.80847
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"kato": "596600000",
|
||||
"district": "Район им. Габита Мусрепова",
|
||||
"Y2022": 42858.7,
|
||||
"Y2023": 43671.1,
|
||||
"Y2024": 44483.5
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
67.02892,
|
||||
52.78419
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
20
test/acceptance/data/style.sld
Normal file
20
test/acceptance/data/style.sld
Normal file
@@ -0,0 +1,20 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<sld:StyledLayerDescriptor version="1.0.0" xmlns:sld="http://www.opengis.net/sld" xmlns:ogc="http://www.opengis.net/ogc">
|
||||
<sld:NamedLayer>
|
||||
<sld:Name>ndvi</sld:Name>
|
||||
<sld:UserStyle>
|
||||
<sld:Title>NDVI gradient</sld:Title>
|
||||
<sld:FeatureTypeStyle>
|
||||
<sld:Rule>
|
||||
<sld:RasterSymbolizer>
|
||||
<sld:ColorMap>
|
||||
<sld:ColorMapEntry color="#a50026" quantity="0.0"/>
|
||||
<sld:ColorMapEntry color="#ffffbf" quantity="0.5"/>
|
||||
<sld:ColorMapEntry color="#006837" quantity="1.0"/>
|
||||
</sld:ColorMap>
|
||||
</sld:RasterSymbolizer>
|
||||
</sld:Rule>
|
||||
</sld:FeatureTypeStyle>
|
||||
</sld:UserStyle>
|
||||
</sld:NamedLayer>
|
||||
</sld:StyledLayerDescriptor>
|
||||
1
test/acceptance/data/style.txt
Normal file
1
test/acceptance/data/style.txt
Normal file
@@ -0,0 +1 @@
|
||||
not an SLD file
|
||||
758
test/acceptance/functional.sh
Executable file
758
test/acceptance/functional.sh
Executable file
@@ -0,0 +1,758 @@
|
||||
#!/usr/bin/env bash
|
||||
# Functional acceptance test of the GIS service on a test stand — the 89 checks
|
||||
# of section 5 of «Протокол функционального тестирования» (Приложение 7):
|
||||
# F01–F38 positive scenarios (categories, datasets, groups, SLD, filters,
|
||||
# PostGIS / MinIO confirmation)
|
||||
# N01–N38 negative scenarios
|
||||
# L02–L05, L07–L15 Landscape Program (ЛП) on synthetic rasters, incl. the
|
||||
# pixel-by-pixel reference check (lp_verify.py)
|
||||
#
|
||||
# Usage:
|
||||
# test/acceptance/functional.sh # stand at http://localhost:18080
|
||||
# BASE_URL=http://host:port test/acceptance/functional.sh
|
||||
#
|
||||
# Environment:
|
||||
# BASE_URL API address (default http://localhost:18080)
|
||||
# LOCAL_CHECKS 1 = confirm in PostGIS/MinIO through the docker containers of
|
||||
# the compose project (F35 MinIO part, F36, F37, F38, L15
|
||||
# resource figures); 0 = skip those (reported as SKIP);
|
||||
# auto (default) = 1 when the gis-acceptance containers run
|
||||
# COMPOSE_PROJECT compose project of the stand (default gis-acceptance)
|
||||
# PYTHON python3 with numpy + rasterio (default python3)
|
||||
# KEEP_DATA 1 = do not delete the created test data at the end
|
||||
# WAIT_TIMEOUT seconds to wait for asynchronous processing (default 300)
|
||||
#
|
||||
# Output: one line per check "ID | описание | PASS/FAIL/SKIP | detail", then a
|
||||
# summary. Exit code 0 only when there is no FAIL. Every object the script
|
||||
# creates carries a unique run prefix and is deleted at the end (unless
|
||||
# KEEP_DATA=1); pre-existing objects are never modified or deleted. The script
|
||||
# writes the ЛП slots, so run it on an isolated test stand, never on production.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
DATA="$HERE/data"
|
||||
BASE_URL="${BASE_URL:-http://localhost:18080}"
|
||||
BASE_URL="${BASE_URL%/}"
|
||||
COMPOSE_PROJECT="${COMPOSE_PROJECT:-gis-acceptance}"
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
KEEP_DATA="${KEEP_DATA:-0}"
|
||||
WAIT_TIMEOUT="${WAIT_TIMEOUT:-300}"
|
||||
LOCAL_CHECKS="${LOCAL_CHECKS:-auto}"
|
||||
|
||||
for bin in curl jq "$PYTHON"; do
|
||||
command -v "$bin" >/dev/null 2>&1 || { echo "missing required tool: $bin" >&2; exit 2; }
|
||||
done
|
||||
"$PYTHON" -c 'import numpy, rasterio' 2>/dev/null || { echo "$PYTHON needs numpy and rasterio (pip install -r $HERE/requirements.txt)" >&2; exit 2; }
|
||||
|
||||
container_of() { # service -> container id of the compose project (empty if none)
|
||||
docker ps -q --filter "label=com.docker.compose.project=$COMPOSE_PROJECT" \
|
||||
--filter "label=com.docker.compose.service=$1" 2>/dev/null | head -n1
|
||||
}
|
||||
if [ "$LOCAL_CHECKS" = auto ]; then
|
||||
LOCAL_CHECKS=0
|
||||
if command -v docker >/dev/null 2>&1 && [ -n "$(container_of postgres)" ] && [ -n "$(container_of minio)" ]; then
|
||||
LOCAL_CHECKS=1
|
||||
fi
|
||||
fi
|
||||
PG_C=""; MINIO_C=""; WORKER_C=""
|
||||
if [ "$LOCAL_CHECKS" = 1 ]; then
|
||||
PG_C="$(container_of postgres)"; MINIO_C="$(container_of minio)"; WORKER_C="$(container_of worker)"
|
||||
fi
|
||||
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
P="acc$(date +%s)" # unique slug prefix of every object this run creates
|
||||
TMP="$(mktemp -d "${TMPDIR:-/tmp}/gis-func.XXXXXX")"
|
||||
RESULTS="$HERE/results/functional-$STAMP"
|
||||
mkdir -p "$RESULTS"
|
||||
BODY="$TMP/body"
|
||||
JSON=(-H 'Content-Type: application/json')
|
||||
|
||||
# ------------------------------------------------------------------ reporting
|
||||
PASSN=0; FAILN=0; SKIPN=0
|
||||
CUR_ID=""; CUR_DESC=""; CUR_OK=1; CUR_MSG=""
|
||||
begin() { CUR_ID="$1"; CUR_DESC="$2"; CUR_OK=1; CUR_MSG=""; }
|
||||
bad() { CUR_OK=0; CUR_MSG="${CUR_MSG:+$CUR_MSG; }$1"; }
|
||||
line() { printf '%-4s | %s | %s | %s\n' "$1" "$2" "$3" "$4" | tee -a "$RESULTS/functional.txt"; }
|
||||
end() { # detail
|
||||
if [ "$CUR_OK" = 1 ]; then PASSN=$((PASSN + 1)); line "$CUR_ID" "$CUR_DESC" PASS "$1"
|
||||
else FAILN=$((FAILN + 1)); line "$CUR_ID" "$CUR_DESC" FAIL "$CUR_MSG"; fi
|
||||
}
|
||||
skip() { SKIPN=$((SKIPN + 1)); line "$CUR_ID" "$CUR_DESC" SKIP "$1"; }
|
||||
snippet() { head -c 220 "$BODY" 2>/dev/null | tr '\n\r' ' '; }
|
||||
|
||||
# ----------------------------------------------------------------------- HTTP
|
||||
CODE=000; TIME=0
|
||||
http() { # METHOD PATH [curl args...] -> $CODE, $TIME (s), body in $BODY
|
||||
local m="$1" p="$2" out; shift 2
|
||||
out="$(curl -sS -o "$BODY" -w '%{http_code} %{time_total}' -X "$m" "$BASE_URL$p" "$@" 2>"$TMP/curl.err")" || out="000 0"
|
||||
CODE="${out%% *}"; TIME="${out#* }"
|
||||
}
|
||||
jb() { jq -r "$1" "$BODY" 2>/dev/null || true; }
|
||||
jbc() { jq -c "${1:-.}" "$BODY" 2>/dev/null || snippet; }
|
||||
need_code() { [ "$CODE" = "$1" ] || bad "HTTP $CODE (ожидалось $1): $(snippet)"; }
|
||||
need_eq() { [ "$1" = "$2" ] || bad "$3: «$1» ≠ «$2»"; }
|
||||
need_has() { case "$1" in *"$2"*) ;; *) bad "$3: нет «$2» в «${1:0:200}»";; esac; }
|
||||
need_true() { [ "$1" = true ] || bad "$2"; }
|
||||
|
||||
sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi; }
|
||||
|
||||
# dataset status polling: wait_ds ID STATUS... -> $WS (final status)
|
||||
WS=""
|
||||
wait_ds() {
|
||||
local id="$1"; shift
|
||||
local deadline=$(( $(date +%s) + WAIT_TIMEOUT ))
|
||||
while :; do
|
||||
WS="$(curl -sS "$BASE_URL/datasets/$id" | jq -r '.status' 2>/dev/null || echo "?")"
|
||||
for s in "$@"; do [ "$WS" = "$s" ] && return 0; done
|
||||
[ "$WS" = failed ] && return 0
|
||||
[ "$(date +%s)" -ge "$deadline" ] && return 0
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
# run polling: wait_run ID -> $RS final status; run JSON in $TMP/run-ID.json
|
||||
RS=""
|
||||
wait_run() {
|
||||
local id="$1" deadline=$(( $(date +%s) + WAIT_TIMEOUT ))
|
||||
while :; do
|
||||
curl -sS "$BASE_URL/landscape/runs/$id" -o "$TMP/run-$id.json" || true
|
||||
RS="$(jq -r '.status' "$TMP/run-$id.json" 2>/dev/null || echo "?")"
|
||||
case "$RS" in succeeded|failed) return 0;; esac
|
||||
[ "$(date +%s)" -ge "$deadline" ] && return 0
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
node_statuses() { jq -r '[.nodes[] | "\(.node_code):\(.status)"] | sort | join(" ")' "$TMP/run-$1.json" 2>/dev/null || true; }
|
||||
wait_outputs() { # RUN: wait until every output dataset of the run is ready (COG done)
|
||||
local id; for id in $(run_outputs "$1" | jq -r '.[].dataset_id'); do wait_ds "$id" ready; done
|
||||
}
|
||||
run_outputs() { jq -c '[.nodes[] | .outputs[]?] | sort_by(.group)' "$TMP/run-$1.json" 2>/dev/null || echo '[]'; }
|
||||
|
||||
# ------------------------------------------------------------- cleanup ledger
|
||||
DATASETS=""; GRPS=""; CATS=""; RUNS=""
|
||||
track_ds() { [ -n "$1" ] && [ "$1" != null ] && DATASETS="$DATASETS $1"; return 0; }
|
||||
cleanup() {
|
||||
local rc=$?
|
||||
if [ "$KEEP_DATA" = 1 ]; then
|
||||
echo "KEEP_DATA=1: test data kept (prefix $P)"
|
||||
else
|
||||
local ids="$DATASETS" r d
|
||||
for r in $RUNS; do # outputs computed by our runs (never reused foreign ones)
|
||||
[ -f "$TMP/run-$r.json" ] || continue
|
||||
ids="$ids $(jq -r '[.nodes[] | select(.status=="succeeded") | .outputs[]?.dataset_id] | join(" ")' "$TMP/run-$r.json" 2>/dev/null || true)"
|
||||
done
|
||||
for d in $ids; do curl -sS -o /dev/null -X DELETE "$BASE_URL/datasets/$d" || true; done
|
||||
for d in $GRPS; do curl -sS -o /dev/null -X DELETE "$BASE_URL/dataset-groups/$d" || true; done
|
||||
for d in $CATS; do curl -sS -o /dev/null -X DELETE "$BASE_URL/categories/$d" || true; done
|
||||
echo "cleanup: removed test datasets, groups and categories with prefix $P (ЛП run records are kept: the API has no run deletion)"
|
||||
fi
|
||||
rm -rf "$TMP"
|
||||
exit "$rc"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "functional acceptance test: $BASE_URL prefix=$P local_checks=$LOCAL_CHECKS log=$RESULTS"
|
||||
: > "$RESULTS/functional.txt"
|
||||
|
||||
# ====================================================================== 5.2
|
||||
begin F01 "Создание корневой категории POST /categories"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-ecology\",\"name\":\"Экология\",\"description\":\"Приёмочный тест\"}"
|
||||
need_code 201; CAT_ECO="$(jb .id)"; need_eq "$(jb .parent_id)" null "parent_id"
|
||||
[ -n "$CAT_ECO" ] && [ "$CAT_ECO" != null ] && CATS="$CAT_ECO $CATS"
|
||||
end "201; $P-ecology «Экология», parent_id null"
|
||||
|
||||
begin F02 "Создание дочерних категорий (parent_id)"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-ecology-soil\",\"name\":\"Почвы\",\"parent_id\":\"$CAT_ECO\"}"
|
||||
need_code 201; CAT_SOIL="$(jb .id)"; need_eq "$(jb .parent_id)" "$CAT_ECO" "parent_id soil"
|
||||
[ -n "$CAT_SOIL" ] && [ "$CAT_SOIL" != null ] && CATS="$CAT_SOIL $CATS"
|
||||
c1="$CODE"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-ecology-soil-humus\",\"name\":\"Гумус\",\"parent_id\":\"$CAT_SOIL\"}"
|
||||
need_code 201; CAT_HUM="$(jb .id)"; need_eq "$(jb .parent_id)" "$CAT_SOIL" "parent_id humus"
|
||||
[ -n "$CAT_HUM" ] && [ "$CAT_HUM" != null ] && CATS="$CAT_HUM $CATS"
|
||||
end "$c1 / $CODE: трёхуровневая иерархия ecology → soil → humus"
|
||||
|
||||
begin F03 "Список категорий с фильтром parent_id"
|
||||
http GET "/categories?parent_id=$CAT_ECO"
|
||||
need_code 200; need_eq "$(jb 'length')" 1 "число дочерних"; need_eq "$(jb '.[0].id')" "$CAT_SOIL" "дочерняя категория"
|
||||
end "200; возвращена дочерняя категория $P-ecology-soil"
|
||||
|
||||
begin F04 "Получение категории по id"
|
||||
http GET "/categories/$CAT_SOIL"
|
||||
need_code 200
|
||||
need_eq "$(jb '[has("id"),has("parent_id"),has("code"),has("name"),has("description")]|all')" true "состав полей"
|
||||
SOIL_UPDATED="$(jb .updated_at)"
|
||||
end "200; поля id, parent_id, code, name, description"
|
||||
|
||||
begin F05 "Обновление категории (PUT)"
|
||||
sleep 1
|
||||
http PUT "/categories/$CAT_SOIL" "${JSON[@]}" -d "{\"code\":\"$P-ecology-soil\",\"name\":\"Почвы и грунты\",\"description\":\"Обновлено\",\"parent_id\":\"$CAT_ECO\"}"
|
||||
need_code 200; need_eq "$(jb .name)" "Почвы и грунты" name; need_eq "$(jb .description)" "Обновлено" description
|
||||
[ "$(jb .updated_at)" != "$SOIL_UPDATED" ] || bad "updated_at не изменился"
|
||||
end "200; name «Почвы и грунты», description «Обновлено», updated_at обновлён"
|
||||
|
||||
begin F06 "Цикл в иерархии (корень → внук)"
|
||||
http PUT "/categories/$CAT_ECO" "${JSON[@]}" -d "{\"code\":\"$P-ecology\",\"name\":\"Экология\",\"parent_id\":\"$CAT_HUM\"}"
|
||||
need_code 422; need_has "$(jb .error)" "own ancestor" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
|
||||
begin F07 "Категория-родитель самой себя"
|
||||
http PUT "/categories/$CAT_ECO" "${JSON[@]}" -d "{\"code\":\"$P-ecology\",\"name\":\"Экология\",\"parent_id\":\"$CAT_ECO\"}"
|
||||
need_code 422; need_has "$(jb .error)" "own ancestor" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
|
||||
begin F08 "Дубликат кода категории"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-ecology\",\"name\":\"Дубликат\"}"
|
||||
need_code 409
|
||||
[ "$CODE" = 201 ] && CATS="$(jb .id) $CATS"
|
||||
end "409 «$(jb .error)»"
|
||||
|
||||
begin F09 "Удаление категории"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-tmp\",\"name\":\"Временная\"}"
|
||||
need_code 201; tmpc="$(jb .id)"; c1="$CODE"
|
||||
http DELETE "/categories/$tmpc"; need_code 204; c2="$CODE"
|
||||
http GET "/categories/$tmpc"; need_code 404
|
||||
end "$c1 (создана) → $c2 (удалена) → повторный GET $CODE"
|
||||
|
||||
# ====================================================================== 5.3
|
||||
begin F10 "Создание группы наборов POST /dataset-groups"
|
||||
http POST /dataset-groups "${JSON[@]}" -d "{\"code\":\"$P-population-nko\",\"name\":\"Численность населения СКО\",\"description\":\"Приёмочный тест\"}"
|
||||
need_code 201; GRP="$(jb .id)"
|
||||
[ -n "$GRP" ] && [ "$GRP" != null ] && GRPS="$GRP $GRPS"
|
||||
end "201; группа $P-population-nko «Численность населения СКО»"
|
||||
|
||||
begin F11 "Загрузка набора vector (GeoJSON)"
|
||||
http POST /datasets -F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F "category_id=$CAT_ECO" \
|
||||
-F "name=Участки СКО" -F "description=Полигоны участков" -F "unit=ha"
|
||||
need_code 201; DS_PLOTS="$(jb .id)"; track_ds "$DS_PLOTS"; s0="$(jb .status)"; key="$(jb .storage_key)"
|
||||
need_has "$key" "/plots_nko.geojson" storage_key
|
||||
wait_ds "$DS_PLOTS" ready; need_eq "$WS" ready status
|
||||
end "201, status $s0 → $WS; storage_key …/plots_nko.geojson"
|
||||
|
||||
begin F12 "Выгрузка vector в GeoJSON"
|
||||
http GET "/datasets/$DS_PLOTS.geojson"
|
||||
need_code 200; need_eq "$(jb .type)" FeatureCollection type; need_eq "$(jb '.features|length')" 4 "число объектов"
|
||||
need_eq "$(jb '[.features[].geometry.type]|unique|join(",")')" Polygon "тип геометрии"
|
||||
ext="$(jb '[.. | arrays | select(length==2 and (.[0]|type)=="number")] as $p | "\([$p[][0]]|min),\([$p[][1]]|min) – \([$p[][0]]|max),\([$p[][1]]|max)"')"
|
||||
need_eq "$(jb '[.. | arrays | select(length==2 and (.[0]|type)=="number")] as $p | ([$p[][0]]|min)==69 and ([$p[][1]]|min)==53.6 and ([$p[][0]]|max)==71.7 and ([$p[][1]]|max)==55')" true "охват (69.0, 53.6) – (71.7, 55.0): $ext"
|
||||
end "200; FeatureCollection, 4 объекта Polygon; охват ($ext)"
|
||||
|
||||
begin F13 "Загрузка vector_with_kato (GeoJSON, реальные коды КАТО)"
|
||||
http POST /datasets -F "file=@$DATA/population_kato.geojson" -F file_type=vector_with_kato -F "category_id=$CAT_ECO" \
|
||||
-F "name=Численность населения" -F unit=чел -F "dg_id=$GRP" -F start_date=2022-01-01 -F end_date=2024-12-31
|
||||
need_code 201; DS_POP="$(jb .id)"; track_ds "$DS_POP"; s0="$(jb .status)"; need_eq "$(jb .dg_id)" "$GRP" dg_id
|
||||
wait_ds "$DS_POP" awaiting_mapping; need_eq "$WS" awaiting_mapping status
|
||||
end "201, status $s0 → $WS, dg_id = группа $P-population-nko"
|
||||
|
||||
begin F14 "Обнаружение колонок"
|
||||
http GET "/datasets/$DS_POP"
|
||||
need_code 200; need_eq "$(jb .status)" awaiting_mapping status
|
||||
cols="$(jb '[.attribute_columns[].name]|join(",")')"
|
||||
need_eq "$cols" "kato,district,Y2022,Y2023,Y2024" колонки
|
||||
need_eq "$(jb '[.attribute_columns[]|(.samples|length)>0]|all')" true "примеры значений"
|
||||
end "200; колонки $cols с примерами значений"
|
||||
|
||||
MAP="$TMP/map.json"
|
||||
mapping() { http POST "/datasets/$DS_POP/mapping" "${JSON[@]}" -d "$1"; }
|
||||
begin N01 "Маппинг: несуществующая колонка года"
|
||||
mapping '{"kato_column":"kato","year_columns":[{"column":"Y1999","start_date":"2022-01-01"}]}'
|
||||
need_code 422; need_has "$(jb .error)" 'year column "Y1999" is not among the detected columns' сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N02 "Маппинг: несуществующая колонка КАТО"
|
||||
mapping '{"kato_column":"KATO_X","year_columns":[{"column":"Y2022","start_date":"2022-01-01"}]}'
|
||||
need_code 422; need_has "$(jb .error)" 'kato_column "KATO_X" is not among the detected columns' сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N03 "Маппинг: end_date раньше start_date"
|
||||
mapping '{"kato_column":"kato","year_columns":[{"column":"Y2022","start_date":"2022-12-31","end_date":"2022-01-01"}]}'
|
||||
need_code 422; need_has "$(jb .error)" 'is before start_date' сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N04 "Маппинг: некорректная дата 2022-13-45"
|
||||
mapping '{"kato_column":"kato","year_columns":[{"column":"Y2022","start_date":"2022-13-45"}]}'
|
||||
need_code 422; need_eq "$(jb .errors.StartDate)" "is invalid" errors.StartDate
|
||||
end "422 $(jbc)"
|
||||
begin N05 "Маппинг: пустой список year_columns"
|
||||
mapping '{"kato_column":"kato","year_columns":[]}'
|
||||
need_code 422; need_has "$(jb .errors.YearColumns)" "at least 1" errors.YearColumns
|
||||
end "422 $(jbc)"
|
||||
|
||||
begin F15 "Сохранение маппинга КАТО и годовых колонок с периодами"
|
||||
mapping '{"kato_column":"kato","year_columns":[{"column":"Y2022","start_date":"2022-01-01","end_date":"2022-12-31"},{"column":"Y2023","start_date":"2023-01-01","end_date":"2023-12-31"},{"column":"Y2024","start_date":"2024-01-01","end_date":"2024-12-31"}]}'
|
||||
need_code 200; s0="$(jb .status)"; need_eq "$(jb '.year_columns|length')" 3 year_columns; need_eq "$(jb .kato_column)" kato kato_column
|
||||
wait_ds "$DS_POP" ready; need_eq "$WS" ready status
|
||||
end "200, status $s0 → $WS; 3 колонки Y2022–Y2024 с периодами"
|
||||
|
||||
fixval() { jq -r --arg k "$1" --arg y "$2" '.features[]|select(.properties.kato==$k)|.properties[$y]' "$DATA/population_kato.geojson"; }
|
||||
begin F16 "Наблюдения (long format)"
|
||||
http GET "/datasets/$DS_POP/observations?page_size=100"
|
||||
need_code 200; need_eq "$(jb .total)" 42 total
|
||||
exp="$(fixval 591000000 Y2022)"
|
||||
got="$(jb '.data[]|select(.kato_code=="591000000" and .start_date=="2022-01-01")|.value')"
|
||||
need_eq "$got" "$exp" "591000000 / 2022"
|
||||
end "200; total $(jb .total) (14 районов × 3 года), 591000000 / 2022 → $got"
|
||||
|
||||
begin F17 "Фильтр наблюдений по kato_code"
|
||||
http GET "/datasets/$DS_POP/observations?kato_code=595000000"
|
||||
need_code 200; need_eq "$(jb .total)" 3 total
|
||||
got="$(jb '[.data|sort_by(.start_date)[]|.value]|join(" / ")')"
|
||||
exp="$(fixval 595000000 Y2022) / $(fixval 595000000 Y2023) / $(fixval 595000000 Y2024)"
|
||||
need_eq "$got" "$exp" "значения 595000000"
|
||||
end "200; для 595000000 — $(jb .total) наблюдения: $got"
|
||||
|
||||
begin F18 "GeoJSON vector_with_kato: соединение с districts"
|
||||
http GET "/datasets/$DS_POP.geojson"
|
||||
need_code 200; need_eq "$(jb '.features|length')" 14 "число объектов"
|
||||
need_eq "$(jb '[.features[].geometry.type]|unique|join(",")')" MultiPolygon "тип геометрии"
|
||||
need_eq "$(jb '.features[0].properties|keys|join(",")')" "data,kato,name,periods" "свойства"
|
||||
end "200; 14 объектов MultiPolygon, свойства data, kato, name, periods"
|
||||
|
||||
begin F19 "Загрузка vector_with_kato (GeoPackage)"
|
||||
http POST /datasets -F "file=@$DATA/livestock_kato.gpkg" -F file_type=vector_with_kato -F "category_id=$CAT_ECO" \
|
||||
-F "name=Поголовье КРС" -F unit=голов
|
||||
need_code 201; DS_LIVE="$(jb .id)"; track_ds "$DS_LIVE"; c1="$CODE"
|
||||
wait_ds "$DS_LIVE" awaiting_mapping; need_eq "$WS" awaiting_mapping status
|
||||
http GET "/datasets/$DS_LIVE"; cols="$(jb '[.attribute_columns[].name]|join(",")')"
|
||||
need_eq "$cols" "fid,KATO_CODE,district,H_2022,H_2023,H_2024" колонки
|
||||
http POST "/datasets/$DS_LIVE/mapping" "${JSON[@]}" -d '{"kato_column":"KATO_CODE","year_columns":[{"column":"H_2022","start_date":"2022-01-01","end_date":"2022-12-31"},{"column":"H_2023","start_date":"2023-01-01","end_date":"2023-12-31"},{"column":"H_2024","start_date":"2024-01-01","end_date":"2024-12-31"}]}'
|
||||
need_code 200; c2="$CODE"
|
||||
wait_ds "$DS_LIVE" ready; need_eq "$WS" ready status
|
||||
end "$c1 → awaiting_mapping; колонки $cols; маппинг $c2, статус $WS"
|
||||
|
||||
NDVI="$TMP/ndvi_nko.tif"
|
||||
"$PYTHON" "$HERE/lib/helpers.py" ndvi "$NDVI"
|
||||
begin F20 "Загрузка растра GeoTIFF (600×400, EPSG:4326, Float32)"
|
||||
http POST /datasets -F "file=@$NDVI" -F file_type=raster -F "category_id=$CAT_ECO" -F "name=NDVI СКО" -F unit=NDVI
|
||||
need_code 201; DS_NDVI="$(jb .id)"; track_ds "$DS_NDVI"; s0="$(jb .status)"
|
||||
wait_ds "$DS_NDVI" ready; need_eq "$WS" ready status
|
||||
end "201, status $s0 → $WS"
|
||||
|
||||
begin F21 "Растр: статус, bbox"
|
||||
http GET "/datasets/$DS_NDVI"
|
||||
need_code 200; need_eq "$(jb .status)" ready status
|
||||
need_has "$(jb .cog_storage_key)" "/cog/ndvi_nko.tif" cog_storage_key
|
||||
bbox="$(jb '.bbox|map(.*1000|round/1000)|join(", ")')"
|
||||
need_eq "$bbox" "68.5, 51.5, 74.5, 55.5" bbox
|
||||
end "200; ready, cog_storage_key …/cog/ndvi_nko.tif, bbox [$bbox]"
|
||||
|
||||
begin F22 "Получение COG и проверка LAYOUT=COG"
|
||||
ctype="$(curl -sS -o "$TMP/ndvi.cog.tif" -w '%{http_code} %{content_type}' "$BASE_URL/datasets/$DS_NDVI.cog" || echo 000)"
|
||||
need_eq "$ctype" "200 image/tiff" "код и Content-Type"
|
||||
ri="$("$PYTHON" "$HERE/lib/helpers.py" rinfo "$TMP/ndvi.cog.tif" 2>&1 || true)"
|
||||
need_eq "$(echo "$ri" | jq -r '"\(.driver) \(.width)x\(.height) EPSG:\(.epsg) \(.layout) \(.compression)"' 2>/dev/null)" "GTiff 600x400 EPSG:4326 COG DEFLATE" "формат"
|
||||
end "$ctype, $(wc -c < "$TMP/ndvi.cog.tif" | tr -d ' ') байт; $(echo "$ri" | jq -r '"\(.driver), \(.width)×\(.height), EPSG:\(.epsg), LAYOUT=\(.layout), \(.compression)"' 2>/dev/null)"
|
||||
|
||||
begin F23 "Скачивание оригинала /download"
|
||||
dl="$(curl -sS -o "$TMP/ndvi.dl.tif" -w '%{http_code}' "$BASE_URL/datasets/$DS_NDVI/download" || echo 000)"
|
||||
need_eq "$dl" 200 "HTTP"
|
||||
h1="$(sha256 "$NDVI")"; h2="$(sha256 "$TMP/ndvi.dl.tif")"
|
||||
need_eq "$h2" "$h1" "SHA-256"
|
||||
end "$dl, $(wc -c < "$TMP/ndvi.dl.tif" | tr -d ' ') байт; SHA-256 совпадает (${h1:0:8}…${h1: -5})"
|
||||
|
||||
begin F24 "Загрузка SLD для набора"
|
||||
http GET "/datasets/$DS_NDVI/sld"; need_code 404; c0="$CODE"
|
||||
http POST "/datasets/$DS_NDVI/sld" -F "file=@$DATA/style.sld"; need_code 200; need_eq "$(jb .sld_filename)" style.sld sld_filename; c1="$CODE"
|
||||
http GET "/datasets/$DS_NDVI/sld"; need_code 200; need_has "$(cat "$BODY")" "NDVI gradient" "SLD"
|
||||
end "до загрузки GET /sld → $c0; POST → $c1, sld_filename style.sld; GET /sld → $CODE («NDVI gradient»)"
|
||||
|
||||
begin F25 "Общий SLD группы и наследование набором"
|
||||
http POST "/dataset-groups/$GRP/sld" -F "file=@$DATA/group_style.sld"; need_code 200; c1="$CODE"
|
||||
http GET "/dataset-groups/$GRP/sld"; need_code 200; need_has "$(cat "$BODY")" "Group fallback style" "SLD группы"
|
||||
http GET "/datasets/$DS_POP/sld"; need_code 200; need_has "$(cat "$BODY")" "Group fallback style" "SLD набора без собственного"
|
||||
end "POST группы → $c1; GET /sld группы и набора без собственного SLD → «Group fallback style»"
|
||||
|
||||
begin N06 "SLD: недопустимое расширение файла"
|
||||
http POST "/datasets/$DS_NDVI/sld" -F "file=@$DATA/style.txt"
|
||||
need_code 422; need_has "$(jb .error)" "sld file must have one of these extensions" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
|
||||
begin F26 "PATCH набора: dg_id и период"
|
||||
http PATCH "/datasets/$DS_NDVI" "${JSON[@]}" -d "{\"dg_id\":\"$GRP\",\"start_date\":\"2026-01-01\",\"end_date\":\"2026-03-31\",\"description\":\"NDVI за I квартал 2026\"}"
|
||||
need_code 200; need_eq "$(jb .dg_id)" "$GRP" dg_id; need_eq "$(jb .start_date)" 2026-01-01 start_date
|
||||
need_eq "$(jb .end_date)" 2026-03-31 end_date; need_eq "$(jb .description)" "NDVI за I квартал 2026" description
|
||||
end "200; dg_id, start_date 2026-01-01, end_date 2026-03-31, description обновлены"
|
||||
|
||||
begin F27 "PUT набора: полная замена метаданных"
|
||||
http PUT "/datasets/$DS_PLOTS" "${JSON[@]}" -d "{\"category_id\":\"$CAT_ECO\",\"name\":\"Участки СКО (ред.)\",\"description\":\"Полигоны участков\",\"unit\":\"га\",\"dg_id\":\"$GRP\",\"start_date\":\"2025-01-01\",\"end_date\":\"2025-12-31\"}"
|
||||
need_code 200; need_eq "$(jb .name)" "Участки СКО (ред.)" name; need_eq "$(jb .unit)" "га" unit
|
||||
need_eq "$(jb .dg_id)" "$GRP" dg_id; need_eq "$(jb .end_date)" 2025-12-31 end_date
|
||||
end "200; name «Участки СКО (ред.)», unit га, dg_id, период 2025"
|
||||
|
||||
begin N07 "PATCH: end_date раньше start_date"
|
||||
http PATCH "/datasets/$DS_PLOTS" "${JSON[@]}" -d '{"start_date":"2025-12-31","end_date":"2025-01-01"}'
|
||||
need_code 422; need_has "$(jb .error)" 'end_date "2025-01-01" is before start_date "2025-12-31"' сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N08 "PATCH: нередактируемое поле file_type"
|
||||
http PATCH "/datasets/$DS_PLOTS" "${JSON[@]}" -d '{"file_type":"raster"}'
|
||||
need_code 422; need_eq "$(jb .error)" "unknown field: file_type" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N09 "PUT: отсутствуют обязательные поля"
|
||||
http PUT "/datasets/$DS_PLOTS" "${JSON[@]}" -d '{"name":"Только имя"}'
|
||||
need_code 422
|
||||
need_eq "$(jb '[.errors.CategoryID,.errors.Description,.errors.Unit]|join(",")')" "is required,is required,is required" errors
|
||||
end "422 CategoryID, Description, Unit «is required»"
|
||||
|
||||
begin F28 "Свежайший набор группы /latest (по max end_date, фильтры file_type, status)"
|
||||
http GET "/dataset-groups/$GRP/latest"; need_code 200; need_eq "$(jb .id)" "$DS_NDVI" "latest"; l1="$(jb .end_date)"
|
||||
http GET "/dataset-groups/$GRP/latest?file_type=vector_with_kato"; need_code 200; need_eq "$(jb .id)" "$DS_POP" "latest vector_with_kato"; l2="$(jb .end_date)"
|
||||
http GET "/dataset-groups/$GRP/latest?status=failed"; need_code 404
|
||||
end "200; NDVI (end_date $l1); с file_type=vector_with_kato — набор населения ($l2); status=failed → $CODE"
|
||||
|
||||
begin F29 "Обновление группы"
|
||||
http PUT "/dataset-groups/$GRP" "${JSON[@]}" -d "{\"code\":\"$P-population-nko\",\"name\":\"Население и NDVI СКО\",\"description\":\"Приёмочный тест\"}"
|
||||
need_code 200; need_eq "$(jb .name)" "Население и NDVI СКО" name
|
||||
end "200; name «Население и NDVI СКО»"
|
||||
|
||||
begin F30 "Удаление группы, на которую ссылаются наборы"
|
||||
http DELETE "/dataset-groups/$GRP"
|
||||
need_code 409
|
||||
end "409 «$(jb .error)»"
|
||||
|
||||
begin F31 "Список групп с фильтром code"
|
||||
http GET "/dataset-groups?code=$P-population-nko"
|
||||
need_code 200; need_eq "$(jb .total)" 1 total; need_eq "$(jb '.data[0].id')" "$GRP" группа
|
||||
end "200; total $(jb .total)"
|
||||
|
||||
begin F32 "Удаление пустой группы"
|
||||
http POST /dataset-groups "${JSON[@]}" -d "{\"code\":\"$P-empty\",\"name\":\"Пустая\"}"
|
||||
need_code 201; eg="$(jb .id)"; c1="$CODE"
|
||||
http DELETE "/dataset-groups/$eg"; need_code 204
|
||||
[ "$CODE" = 204 ] || GRPS="$eg $GRPS"
|
||||
end "$c1 → $CODE"
|
||||
|
||||
begin N10 "Группа: код не slug («Bad Code!»)"
|
||||
http POST /dataset-groups "${JSON[@]}" -d '{"code":"Bad Code!","name":"Плохой код"}'
|
||||
need_code 422; need_eq "$(jb .errors.Code)" "is invalid" errors.Code
|
||||
[ "$CODE" = 201 ] && GRPS="$(jb .id) $GRPS"
|
||||
end "422 $(jbc)"
|
||||
|
||||
begin F33 "Список наборов: пагинация"
|
||||
http GET "/datasets?category_id=$CAT_ECO&page_size=2&page=1"
|
||||
need_code 200; need_eq "$(jb .total)" 4 total; need_eq "$(jb .page_size)" 2 page_size; need_eq "$(jb .total_pages)" 2 total_pages
|
||||
p1="$(jb '[.data[].id]|join(" ")')"
|
||||
http GET "/datasets?category_id=$CAT_ECO&page_size=2&page=2"
|
||||
need_code 200; p2="$(jb '[.data[].id]|join(" ")')"
|
||||
all="$(printf '%s\n' $p1 $p2 | sort -u | wc -l | tr -d ' ')"
|
||||
need_eq "$all" 4 "уникальных записей на 2 страницах"
|
||||
end "200; total 4, page_size 2, total_pages 2; страницы 1 и 2 без пересечений"
|
||||
|
||||
begin F34 "Фильтры dg_id + end_date_from, file_type + status, category_id + start_date_to"
|
||||
http GET "/datasets?dg_id=$GRP&end_date_from=2025-01-01"; need_code 200; n1="$(jb .total)"
|
||||
need_eq "$(jb '[.data[].id]|sort|join(" ")')" "$(printf '%s\n' "$DS_PLOTS" "$DS_NDVI" | sort | tr '\n' ' ' | sed 's/ $//')" "dg_id+end_date_from"
|
||||
http GET "/datasets?category_id=$CAT_ECO&file_type=vector_with_kato&status=ready"; need_code 200; n2="$(jb .total)"
|
||||
need_eq "$(jb '[.data[].id]|sort|join(" ")')" "$(printf '%s\n' "$DS_POP" "$DS_LIVE" | sort | tr '\n' ' ' | sed 's/ $//')" "file_type+status"
|
||||
http GET "/datasets?category_id=$CAT_ECO&start_date_to=2024-01-01"; need_code 200; n3="$(jb .total)"
|
||||
need_eq "$(jb '[.data[].id]|join(" ")')" "$DS_POP" "category_id+start_date_to"
|
||||
end "200; $n1 / $n2 / $n3 записи соответственно, состав соответствует условиям"
|
||||
|
||||
begin N11 "Фильтр: некорректная дата 2025-31-12"
|
||||
http GET "/datasets?start_date_from=2025-31-12"
|
||||
need_code 400; need_eq "$(jb .error)" "start_date_from must be YYYY-MM-DD" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
begin N12 "Пагинация: page_size=0"
|
||||
http GET "/datasets?page_size=0"
|
||||
need_code 400; need_eq "$(jb .error)" "invalid page_size" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
begin N13 "Фильтр: file_type=shapefile"
|
||||
http GET "/datasets?file_type=shapefile"
|
||||
need_code 400; need_has "$(jb .error)" "file_type must be one of" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
|
||||
# ====================================================================== 5.4
|
||||
printf 'just text\n' > "$TMP/notes.txt"
|
||||
printf 'this is not a tiff\n' > "$TMP/fake.tif"
|
||||
printf 'not json at all\n' > "$TMP/bad.geojson"
|
||||
up() { http POST /datasets "$@"; [ "$CODE" = 201 ] && track_ds "$(jb .id)"; return 0; }
|
||||
negup() { # ID desc expected-code expected-substring curl-args...
|
||||
local id="$1" desc="$2" ec="$3" sub="$4"; shift 4
|
||||
begin "$id" "$desc"; up "$@"; need_code "$ec"; need_has "$(jb .error)" "$sub" сообщение
|
||||
end "$CODE «$(jb .error)»"
|
||||
}
|
||||
negup N14 "Загрузка: расширение .txt" 422 'extension ".txt" is not allowed for file_type "vector"' \
|
||||
-F "file=@$TMP/notes.txt" -F file_type=vector -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N15 "Загрузка: .tif без TIFF-сигнатуры" 422 "file is not a valid TIFF/BigTIFF GeoTIFF" \
|
||||
-F "file=@$TMP/fake.tif" -F file_type=raster -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N16 "Загрузка: .geojson не JSON" 422 "file is not valid GeoJSON" \
|
||||
-F "file=@$TMP/bad.geojson" -F file_type=vector -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N17 "Загрузка: неизвестный file_type" 422 "file_type must be one of: vector_with_kato, vector, raster" \
|
||||
-F "file=@$DATA/plots_nko.geojson" -F file_type=shapefile -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N18 "Загрузка: .tif для типа vector" 422 'extension ".tif" is not allowed for file_type "vector"' \
|
||||
-F "file=@$NDVI" -F file_type=vector -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N19 "Загрузка: нет файла" 400 "file is required" \
|
||||
-F file_type=vector -F "category_id=$CAT_SOIL" -F name=x
|
||||
negup N20 "Загрузка: нет category_id" 422 "category_id must be a valid UUID" \
|
||||
-F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F name=x
|
||||
|
||||
begin N21 "Загрузка без name (поле необязательно по OpenAPI)"
|
||||
up -F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F "category_id=$CAT_SOIL"
|
||||
need_code 201; DS_N21="$(jb .id)"; N21_KEY="$(jb .storage_key)"; need_eq "$(jb .name)" plots_nko.geojson name
|
||||
end "201; name = имя файла ($(jb .name))"
|
||||
|
||||
negup N22 "Загрузка: end_date раньше start_date" 422 'end_date "2025-01-01" is before start_date "2025-12-31"' \
|
||||
-F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F "category_id=$CAT_SOIL" -F name=x -F start_date=2025-12-31 -F end_date=2025-01-01
|
||||
negup N23 "Загрузка: несуществующая категория" 422 "category does not exist" \
|
||||
-F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F "category_id=00000000-0000-4000-8000-000000000000" -F name=x
|
||||
negup N24 "Загрузка: невалидный JSON в meta" 422 "meta must be valid JSON" \
|
||||
-F "file=@$DATA/plots_nko.geojson" -F file_type=vector -F "category_id=$CAT_SOIL" -F name=x -F 'meta={bad json'
|
||||
begin N25 "Загрузка: не multipart"
|
||||
http POST /datasets "${JSON[@]}" -d '{"file_type":"vector"}'
|
||||
need_code 400; need_eq "$(jb .error)" "request must be multipart/form-data" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
|
||||
begin N26 "Несуществующие id на 10 эндпоинтах"
|
||||
X=7d3c9a52-1f4e-4b8a-9c2d-5e6f7a8b9c0d; codes=""
|
||||
for p in "/datasets/$X" "/datasets/$X.geojson" "/datasets/$X.cog" "/datasets/$X/download" "/datasets/$X/observations" \
|
||||
"/datasets/$X/sld" "/categories/$X" "/dataset-groups/$X" "/dataset-groups/$X/latest" "/landscape/runs/$X"; do
|
||||
http GET "$p"; codes="$codes $CODE"; [ "$CODE" = 404 ] || bad "$p → $CODE"
|
||||
done
|
||||
end "404 на всех 10 эндпоинтах:$codes"
|
||||
|
||||
begin N27 "Некорректный формат id (4 эндпоинта)"
|
||||
codes=""
|
||||
for p in /datasets/not-a-uuid /categories/not-a-uuid /dataset-groups/not-a-uuid /landscape/runs/not-a-uuid; do
|
||||
http GET "$p"; codes="$codes $CODE"; [ "$CODE" = 400 ] || bad "$p → $CODE"; [ "$(jb .error)" = "invalid id" ] || bad "$p: $(jb .error)"
|
||||
done
|
||||
end "400 «invalid id» на всех 4:$codes"
|
||||
|
||||
begin N28 "Битый JSON"
|
||||
http POST /categories "${JSON[@]}" -d '{"code":"x","name":'
|
||||
need_code 400; need_eq "$(jb .error)" "invalid request body" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
begin N29 "Мусорное тело 200 КБ"
|
||||
head -c 204800 /dev/urandom > "$TMP/garbage.bin"
|
||||
http POST /categories "${JSON[@]}" --data-binary "@$TMP/garbage.bin"
|
||||
need_code 400; need_eq "$(jb .error)" "invalid request body" сообщение
|
||||
end "400 «$(jb .error)»"
|
||||
begin N30 "Неизвестное поле в JSON"
|
||||
http POST /categories "${JSON[@]}" -d "{\"code\":\"$P-unknown\",\"name\":\"x\",\"color\":\"red\"}"
|
||||
need_code 400; need_eq "$(jb .error)" "invalid request body" сообщение
|
||||
[ "$CODE" = 201 ] && CATS="$(jb .id) $CATS"
|
||||
end "400 «$(jb .error)»"
|
||||
begin N37 "JSON-тело 2 200 048 байт (> 1 МиБ)"
|
||||
{ printf '{"code":"%s-big","name":"' "$P"; head -c $((2200048 - 25 - ${#P})) /dev/zero | tr '\0' 'a'; printf '"}'; } > "$TMP/big.json"
|
||||
http POST /categories "${JSON[@]}" --data-binary "@$TMP/big.json"
|
||||
need_code 400; need_eq "$(jb .error)" "invalid request body" сообщение
|
||||
[ "$CODE" = 201 ] && CATS="$(jb .id) $CATS"
|
||||
end "400 «$(jb .error)» (тело $(wc -c < "$TMP/big.json" | tr -d ' ') байт)"
|
||||
|
||||
runreq() { http POST /landscape/runs "${JSON[@]}" -d "$1"; [ "$CODE" = 202 ] && RUNS="$RUNS $(jb .id)"; return 0; }
|
||||
begin N31 "ЛП: from_stage=7"
|
||||
runreq '{"from_stage":7}'; need_code 422; need_eq "$(jb .errors.FromStage)" "must be 3 or less" errors.FromStage
|
||||
end "422 $(jbc)"
|
||||
begin N32 "ЛП: tie_priority не перестановка"
|
||||
runreq '{"params":{"tie_priority":[1,1,2,3,4]}}'; need_code 422; need_has "$(jb .error)" "tie_priority must be a permutation of 1..5" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
begin N33 "ЛП: override с некорректным UUID"
|
||||
runreq '{"overrides":{"lp-z-lcr":"not-a-uuid"}}'; need_code 422; need_eq "$(jb '.errors["Overrides[lp-z-lcr]"]')" "must be a valid UUID" errors
|
||||
end "422 $(jbc)"
|
||||
begin N34 "ЛП: неизвестный слот"
|
||||
http POST /landscape/slots/lp-unknown-slot -F "file=@$NDVI"; need_code 404
|
||||
[ "$CODE" = 201 ] && track_ds "$(jb .id)"
|
||||
end "404 «$(jb .error)»"
|
||||
begin N35 "ЛП: не-GeoTIFF в слот"
|
||||
http POST /landscape/slots/lp-z-lcr -F "file=@$DATA/plots_nko.geojson"; need_code 422
|
||||
need_has "$(jb .error)" 'extension ".geojson" is not allowed for file_type "raster"' сообщение
|
||||
[ "$CODE" = 201 ] && track_ds "$(jb .id)"
|
||||
end "422 «$(jb .error)»"
|
||||
begin N36 "ЛП: window_size=100"
|
||||
runreq '{"params":{"window_size":100}}'; need_code 422; need_has "$(jb .error)" "window_size must be in 256..8192" сообщение
|
||||
end "422 «$(jb .error)»"
|
||||
|
||||
begin N38 "Загрузка 100 МБ мусора с сигнатурой TIFF"
|
||||
{ printf 'II*\000'; head -c 104857600 /dev/urandom; } > "$TMP/garbage.tif"
|
||||
up -F "file=@$TMP/garbage.tif" -F file_type=raster -F "category_id=$CAT_SOIL" -F "name=$P garbage"
|
||||
need_code 201; DS_G="$(jb .id)"; up_time="$TIME"
|
||||
wait_ds "$DS_G" ready; need_eq "$WS" failed status
|
||||
http GET "/datasets/$DS_G"; pe="$(jb .parse_error)"; need_has "$pe" gdal_translate parse_error
|
||||
http GET /readyz; need_code 200; need_eq "$(jb .ready)" true "/readyz ready"
|
||||
rm -f "$TMP/garbage.tif"
|
||||
end "201 (104 857 604 байт за ${up_time} с) → status $WS, parse_error от gdal_translate; /readyz 200, сервис доступен"
|
||||
|
||||
begin F35 "Удаление набора (строка и объект в S3)"
|
||||
http DELETE "/datasets/$DS_N21"; need_code 204; c1="$CODE"
|
||||
http GET "/datasets/$DS_N21"; need_code 404; c2="$CODE"
|
||||
detail="$c1; GET → $c2"
|
||||
if [ "$LOCAL_CHECKS" = 1 ]; then
|
||||
prefix="${N21_KEY%/*}"
|
||||
n="$(docker exec "$MINIO_C" sh -c "MC_CONFIG_DIR=/tmp/.mc MC_HOST_local=http://\$MINIO_ROOT_USER:\$MINIO_ROOT_PASSWORD@127.0.0.1:9000 mc ls --recursive local/geofiles/$prefix/ 2>/dev/null | wc -l" | tr -d ' ' || echo "?")"
|
||||
need_eq "$n" 0 "объектов в префиксе MinIO"
|
||||
detail="$detail; объектов в префиксе MinIO $prefix/ — $n"
|
||||
else
|
||||
detail="$detail; проверка MinIO пропущена (LOCAL_CHECKS=0)"
|
||||
fi
|
||||
end "$detail"
|
||||
|
||||
# ====================================================================== 5.5
|
||||
psql_q() { docker exec -i "$PG_C" psql -U gis -d gis -At -F '|' -c "$1"; }
|
||||
begin F36 "Геометрии наборов в PostGIS: тип, SRID, ST_IsValid"
|
||||
if [ "$LOCAL_CHECKS" = 1 ]; then
|
||||
out="$(psql_q "SELECT id, ST_GeometryType(geometry), ST_SRID(geometry), ST_IsValid(geometry) FROM datasets WHERE id IN ('$DS_PLOTS','$DS_POP','$DS_LIVE','$DS_NDVI')" 2>&1 || true)"
|
||||
echo "$out" > "$RESULTS/F36.txt"
|
||||
for pair in "$DS_PLOTS:ST_MultiPolygon" "$DS_POP:ST_MultiPoint" "$DS_LIVE:ST_MultiPoint" "$DS_NDVI:ST_Polygon"; do
|
||||
id="${pair%%:*}"; want="${pair#*:}"
|
||||
row="$(echo "$out" | grep "^$id|" || true)"
|
||||
need_eq "$(echo "$row" | cut -d'|' -f2-4)" "$want|4326|t" "геометрия $id"
|
||||
done
|
||||
end "участки ST_MultiPolygon; население и КРС ST_MultiPoint; NDVI ST_Polygon; у всех SRID 4326, valid = t"
|
||||
else skip "LOCAL_CHECKS=0 (нет доступа к контейнеру PostGIS)"; fi
|
||||
|
||||
begin F37 "Наблюдения dataset_observations соединяются с districts по КАТО"
|
||||
if [ "$LOCAL_CHECKS" = 1 ]; then
|
||||
out="$(psql_q "SELECT count(*), count(DISTINCT o.kato_code), count(*) FILTER (WHERE d.kato IS NULL) FROM dataset_observations o LEFT JOIN districts d ON d.kato = o.kato_code WHERE o.dataset_id IN ('$DS_POP','$DS_LIVE')" 2>&1 || true)"
|
||||
need_eq "$out" "84|14|0" "наблюдений|кодов КАТО|сирот"
|
||||
kz="$(psql_q "SELECT d.name || ' (' || d.kato || ') — ' || string_agg(o.value::text, ' / ' ORDER BY o.start_date) FROM dataset_observations o JOIN districts d ON d.kato = o.kato_code WHERE o.dataset_id = '$DS_POP' AND o.kato_code = '595000000' GROUP BY d.name, d.kato" 2>&1 || true)"
|
||||
end "84 наблюдения, 14 кодов КАТО, orphan_kato 0 ($out); $kz"
|
||||
else skip "LOCAL_CHECKS=0 (нет доступа к контейнеру PostGIS)"; fi
|
||||
|
||||
begin F38 "Объекты в бакете geofiles MinIO"
|
||||
if [ "$LOCAL_CHECKS" = 1 ]; then
|
||||
keys=""
|
||||
for d in "$DS_PLOTS" "$DS_POP" "$DS_LIVE" "$DS_NDVI"; do
|
||||
http GET "/datasets/$d"; keys="$keys $(jb '[.storage_key, .cog_storage_key, .sld_storage_key] | map(select(. != null)) | join(" ")')"
|
||||
done
|
||||
http GET "/dataset-groups/$GRP"; keys="$keys $(jb .sld_storage_key)"
|
||||
found=0; total=0
|
||||
for k in $keys; do
|
||||
total=$((total + 1))
|
||||
if docker exec "$MINIO_C" sh -c "MC_CONFIG_DIR=/tmp/.mc MC_HOST_local=http://\$MINIO_ROOT_USER:\$MINIO_ROOT_PASSWORD@127.0.0.1:9000 mc stat local/geofiles/$k" >/dev/null 2>&1; then
|
||||
found=$((found + 1))
|
||||
else bad "нет объекта $k"; fi
|
||||
done
|
||||
du="$(docker exec "$MINIO_C" sh -c 'MC_CONFIG_DIR=/tmp/.mc MC_HOST_local=http://$MINIO_ROOT_USER:$MINIO_ROOT_PASSWORD@127.0.0.1:9000 mc du local/geofiles' 2>/dev/null | tr -s ' \t' ' ' || true)"
|
||||
[ "$total" -eq 7 ] || bad "ожидалось 7 ключей (4 исходных файла, COG, SLD набора, SLD группы), получено $total"
|
||||
end "$found/$total объектов найдено: plots_nko.geojson, population_kato.geojson, livestock_kato.gpkg, ndvi_nko.tif, cog/ndvi_nko.tif, sld/style.sld, groups/…/sld/group_style.sld; бакет: $du"
|
||||
else skip "LOCAL_CHECKS=0 (нет доступа к контейнеру MinIO)"; fi
|
||||
|
||||
# ====================================================================== 5.6 ЛП
|
||||
LPIN="$TMP/lpin"; LPCACHE="$TMP/lpcache"; mkdir -p "$LPCACHE"
|
||||
"$PYTHON" "$HERE/lp_gen.py" --out "$LPIN" > "$RESULTS/lp_gen.txt"
|
||||
SLOTS="lp-z-biodiversity lp-z-lcr lp-z-soil-protection lp-z-wr lp-u-surf-water lp-u-soil lp-u-crop lp-u-veg lp-p-irrigation lp-p-snow lp-p-crop lp-p-pasture lp-p-tourism"
|
||||
slot_id_file() { echo "$TMP/slot-$1"; }
|
||||
|
||||
begin L02 "Загрузка 13 входных растров в слоты POST /landscape/slots/{code}"
|
||||
codes=""
|
||||
for s in $SLOTS; do
|
||||
http POST "/landscape/slots/$s" -F "file=@$LPIN/$s.tif" -F "name=$P $s" -F start_date=2025-01-01 -F end_date=2025-12-31
|
||||
codes="$codes $CODE"; [ "$CODE" = 201 ] || bad "$s → $CODE: $(snippet)"
|
||||
id="$(jb .id)"; echo "$id" > "$(slot_id_file "$s")"; track_ds "$id"
|
||||
done
|
||||
for s in $SLOTS; do wait_ds "$(cat "$(slot_id_file "$s")")" ready; [ "$WS" = ready ] || bad "$s: status $WS"; done
|
||||
http GET /landscape/slots
|
||||
for s in $SLOTS; do
|
||||
got="$(jq -r --arg s "$s" '.[]|select(.code==$s)|"\(.latest.id) \(.latest.status)"' "$BODY")"
|
||||
need_eq "$got" "$(cat "$(slot_id_file "$s")") ready" "слот $s"
|
||||
done
|
||||
end "201 для всех 13 слотов; GET /landscape/slots — у всех слотов latest = загруженный растр, status ready"
|
||||
|
||||
begin L03 "Запуск полного прогона {\"from_stage\":0}"
|
||||
runreq '{"from_stage":0}'; need_code 202; RUN1="$(jb .id)"; need_eq "$(jb .status)" queued status
|
||||
need_eq "$(jq -S -c .params "$BODY")" '{"tie_priority":[1,2,3,4,5],"v1_min_s_e12":3,"v1_min_s_e3":4,"v2_min_s_e3":3,"v2_min_s_e4":5,"window_size":2048}' params
|
||||
end "202; run ${RUN1:0:8} поставлен в очередь, params по умолчанию (window_size 2048, tie_priority [1..5], пороги 3/4/3/5)"
|
||||
|
||||
begin L04 "Завершение прогона"
|
||||
wait_run "$RUN1"; need_eq "$RS" succeeded status; ns="$(node_statuses "$RUN1")"
|
||||
need_eq "$ns" "00:succeeded 01:succeeded 02:succeeded 03:succeeded 10:succeeded 20:succeeded 21:succeeded 30:succeeded 31:succeeded" узлы
|
||||
need_eq "$(jq -r .error "$TMP/run-$RUN1.json")" null error
|
||||
end "$RS за $("$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run-$RUN1.json"); error null; все 9 узлов succeeded"
|
||||
|
||||
begin L05 "Результаты GET /landscape/results"
|
||||
outs="$(run_outputs "$RUN1")"
|
||||
# Results list only ready layers; the COG conversion of the outputs is asynchronous.
|
||||
for id in $(jq -r '.[].dataset_id' <<< "$outs"); do wait_ds "$id" ready; [ "$WS" = ready ] || bad "выход $id: status $WS"; done
|
||||
http GET /landscape/results; need_code 200; cp "$BODY" "$TMP/results.json"
|
||||
need_eq "$(jb 'length')" 15 "число групп"
|
||||
for g in $(jq -r '.[].group' <<< "$outs"); do
|
||||
want="$(jq -r --arg g "$g" '.[]|select(.group==$g)|.dataset_id' <<< "$outs")"
|
||||
got="$(jq -r --arg g "$g" '.[]|select(.group==$g)|"\(.latest.id) \(.latest.status) \(.cog_url)"' "$TMP/results.json")"
|
||||
need_eq "$got" "$want ready /datasets/$want.cog" "результат $g"
|
||||
done
|
||||
first="$(jq -r '.[0].dataset_id' <<< "$outs")"
|
||||
http GET "/datasets/$first"; need_eq "$(jb .automated)" true automated; need_eq "$(jb .meta.lp_run_id)" "$RUN1" meta.lp_run_id
|
||||
[ "$(jb .meta.lp_node)" != null ] || bad "meta.lp_node"; [ "$(jb '.meta.inputs|length')" -ge 1 ] 2>/dev/null || bad "meta.inputs"
|
||||
fmt_ok=0
|
||||
for id in $(jq -r '.[].dataset_id' <<< "$outs"); do
|
||||
curl -fsS -o "$LPCACHE/$id.cog.tif" "$BASE_URL/datasets/$id.cog" || rm -f "$LPCACHE/$id.cog.tif"
|
||||
ri="$("$PYTHON" "$HERE/lib/helpers.py" rinfo "$LPCACHE/$id.cog.tif" 2>/dev/null || echo '{}')"
|
||||
f="$(jq -r '"\(.width)x\(.height) EPSG:\(.epsg) \(.layout) \(.dtype) \(.nodata|floor)"' <<< "$ri" 2>/dev/null || true)"
|
||||
if [ "$f" = "3000x2000 EPSG:32642 COG int16 -9999" ]; then fmt_ok=$((fmt_ok + 1)); else bad "COG $id: $f"; fi
|
||||
done
|
||||
end "200; 15 групп (узлы 00…31), все ready, cog_url /datasets/{id}.cog; выходной набор: automated true, meta.lp_run_id, lp_node, inputs. $fmt_ok/15 COG: 3000×2000, EPSG:32642, LAYOUT=COG, Int16, NoData −9999"
|
||||
|
||||
begin L07 "Попиксельная сверка 15 выходов с эталонной реализацией"
|
||||
if "$PYTHON" "$HERE/lp_verify.py" --base-url "$BASE_URL" --run "$RUN1" --cache "$LPCACHE" > "$RESULTS/L07_lp_verify.txt" 2>&1; then :; else bad "lp_verify: $(tail -n 3 "$RESULTS/L07_lp_verify.txt" | tr '\n' ' ')"; fi
|
||||
tot="$(grep 'TOTAL mismatching' "$RESULTS/L07_lp_verify.txt" || echo 'нет итога')"
|
||||
end "15 выходов × 6 000 000 пикселей; «$tot» (журнал: results/functional-$STAMP/L07_lp_verify.txt)"
|
||||
|
||||
begin L08 "Выборочная проверка 9 пикселей (Z_int, U_int, E, C_k, V1–V3)"
|
||||
PTS="20,20;1450,1000;1575,1000;350,350;520,320;500,1500;1000,700;2200,1500;2900,1900"
|
||||
if "$PYTHON" "$HERE/lp_verify.py" --base-url "$BASE_URL" --run "$RUN1" --cache "$LPCACHE" --points "$PTS" > "$RESULTS/L08_points.txt" 2>&1; then :; else bad "$(tail -n 1 "$RESULTS/L08_points.txt")"; fi
|
||||
end "$(tail -n 1 "$RESULTS/L08_points.txt"): NoData (20,20), дыра NoData в U crop (1450,1000), crop и veg NoData (1575,1000), P snow = 0 и 6"
|
||||
|
||||
begin L09 "Повторный прогон {} без изменений входов"
|
||||
runreq '{}'; need_code 202; RUN2="$(jb .id)"; wait_run "$RUN2"; need_eq "$RS" succeeded status
|
||||
ns="$(node_statuses "$RUN2")"
|
||||
need_eq "$ns" "00:skipped 01:skipped 02:skipped 03:skipped 10:skipped 20:skipped 21:skipped 30:skipped 31:skipped" узлы
|
||||
need_eq "$(run_outputs "$RUN2")" "$(run_outputs "$RUN1")" "выходные наборы"
|
||||
end "202; run ${RUN2:0:8} $RS за $("$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run-$RUN2.json"); все 9 узлов skipped, выходные наборы совпадают с первым прогоном"
|
||||
|
||||
begin L10 "Прогон {\"from_stage\":3}"
|
||||
runreq '{"from_stage":3}'; need_code 202; RUN3="$(jb .id)"; wait_run "$RUN3"; need_eq "$RS" succeeded status
|
||||
ns="$(node_statuses "$RUN3")"
|
||||
need_eq "$ns" "00:skipped 01:skipped 02:skipped 03:skipped 10:skipped 20:skipped 21:skipped 30:succeeded 31:succeeded" узлы
|
||||
new="$(jq -r '[.nodes[]|select(.node_code=="30" or .node_code=="31")|.outputs[]?.dataset_id]|join(" ")' "$TMP/run-$RUN3.json" || true)"
|
||||
for id in $new; do run_outputs "$RUN1" | grep -q "$id" && bad "узел стадии 3 переиспользовал $id"; done
|
||||
end "202; run ${RUN3:0:8} $RS за $("$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run-$RUN3.json"); узлы 00–21 skipped, узлы 30 и 31 succeeded (новые выходные наборы)"
|
||||
|
||||
begin L11 "Прогон с параметрами tie_priority [5,4,3,2,1], пороги 4/5/4/4"
|
||||
runreq '{"params":{"tie_priority":[5,4,3,2,1],"v1_min_s_e12":4,"v1_min_s_e3":5,"v2_min_s_e3":4,"v2_min_s_e4":4}}'
|
||||
need_code 202; RUN4="$(jb .id)"; wait_run "$RUN4"; need_eq "$RS" succeeded status
|
||||
ns="$(node_statuses "$RUN4")"
|
||||
need_eq "$ns" "00:skipped 01:skipped 02:skipped 03:skipped 10:skipped 20:skipped 21:skipped 30:succeeded 31:skipped" узлы
|
||||
end "202; run ${RUN4:0:8} $RS за $("$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run-$RUN4.json"); пересчитан только узел 30 (V1–V3), остальные skipped"
|
||||
|
||||
begin L12 "Сверка V1–V3 с пользовательскими параметрами"
|
||||
wait_outputs "$RUN4"
|
||||
if "$PYTHON" "$HERE/lp_verify.py" --base-url "$BASE_URL" --run "$RUN4" --cache "$LPCACHE" > "$RESULTS/L12_lp_verify.txt" 2>&1; then :; else bad "lp_verify: $(tail -n 3 "$RESULTS/L12_lp_verify.txt" | tr '\n' ' ')"; fi
|
||||
v="$(grep -E '^lp-v[123]-' "$RESULTS/L12_lp_verify.txt" | awk '{printf "%s mismatch %s; ", $1, $6}' || true)"
|
||||
end "$v; «$(grep 'TOTAL mismatching' "$RESULTS/L12_lp_verify.txt" || echo 'нет итога')» (все 15 выходов)"
|
||||
|
||||
begin L13 "Невыровненный растр (1500×1000, 200 м) в слоте lp-u-veg"
|
||||
http POST /landscape/slots/lp-u-veg -F "file=@$LPIN/misaligned-lp-u-veg.tif" -F "name=$P misaligned" -F start_date=2025-01-01 -F end_date=2025-12-31
|
||||
need_code 201; DS_MIS="$(jb .id)"; track_ds "$DS_MIS"; wait_ds "$DS_MIS" ready; need_eq "$WS" ready "status растра"
|
||||
runreq '{}'; need_code 202; RUN5="$(jb .id)"; wait_run "$RUN5"; need_eq "$RS" failed status
|
||||
err="$(jq -r .error "$TMP/run-$RUN5.json" || true)"
|
||||
need_has "$err" "input lp-u-veg grid (1500x1000" "сообщение"; need_has "$err" "inputs must be aligned" "сообщение"
|
||||
later="$(jq -r '[.nodes[]|select(.stage>=1)|.status]|unique|join(",")' "$TMP/run-$RUN5.json" || true)"
|
||||
need_eq "$later" pending "узлы стадий 1–3"
|
||||
end "run ${RUN5:0:8} $RS: «${err:0:160}…»; последующие узлы не выполняются ($later)"
|
||||
|
||||
begin L14 "Восстановление: удаление растра, повторный прогон"
|
||||
http DELETE "/datasets/$DS_MIS"; need_code 204; c1="$CODE"
|
||||
http GET /landscape/slots
|
||||
need_eq "$(jq -r '.[]|select(.code=="lp-u-veg")|.latest.id' "$BODY")" "$(cat "$(slot_id_file lp-u-veg)")" "слот lp-u-veg"
|
||||
runreq '{}'; need_code 202; RUN6="$(jb .id)"; wait_run "$RUN6"; need_eq "$RS" succeeded status
|
||||
ns="$(node_statuses "$RUN6")"
|
||||
need_eq "$ns" "00:skipped 01:skipped 02:skipped 03:skipped 10:skipped 20:skipped 21:skipped 30:skipped 31:skipped" узлы
|
||||
end "DELETE $c1; в слоте снова исходный растр; run ${RUN6:0:8} $RS, все узлы skipped"
|
||||
|
||||
begin L15 "Повторный полный прогон с замером ресурсов worker"
|
||||
cpu0=""
|
||||
if [ "$LOCAL_CHECKS" = 1 ] && [ -n "$WORKER_C" ]; then
|
||||
cpu0="$(docker exec "$WORKER_C" sh -c "awk '/usage_usec/{print \$2}' /sys/fs/cgroup/cpu.stat" 2>/dev/null || true)"
|
||||
fi
|
||||
runreq '{"from_stage":0}'; need_code 202; RUN7="$(jb .id)"; c1="$CODE"; wait_run "$RUN7"; need_eq "$RS" succeeded status
|
||||
ns="$(node_statuses "$RUN7")"
|
||||
need_eq "$ns" "00:succeeded 01:succeeded 02:succeeded 03:succeeded 10:succeeded 20:succeeded 21:succeeded 30:succeeded 31:succeeded" узлы
|
||||
res=""
|
||||
if [ -n "$cpu0" ]; then
|
||||
cpu1="$(docker exec "$WORKER_C" sh -c "awk '/usage_usec/{print \$2}' /sys/fs/cgroup/cpu.stat" 2>/dev/null || echo "$cpu0")"
|
||||
peak="$(docker exec "$WORKER_C" sh -c 'cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0')"
|
||||
res="; CPU worker $(awk -v a="$cpu0" -v b="$cpu1" 'BEGIN{printf "%.2f", (b-a)/1e6}') с (включая COG-конвертацию), memory.peak $(awk -v p="$peak" 'BEGIN{printf "%.2f", p/1e9}') ГБ (с учётом страничного кэша)"
|
||||
fi
|
||||
end "$c1; run ${RUN7:0:8} $RS за $("$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run-$RUN7.json")$res"
|
||||
|
||||
# ==================================================================== summary
|
||||
TOTAL=$((PASSN + FAILN + SKIPN))
|
||||
{
|
||||
echo "-----------------------------------------------------------------"
|
||||
echo "ИТОГО: $TOTAL проверок — PASS $PASSN, FAIL $FAILN, SKIP $SKIPN"
|
||||
echo "RESULT: $PASSN/$TOTAL PASS"
|
||||
echo "ЛП прогоны: RUN1=${RUN1:-} (from_stage 0), RUN4=${RUN4:-} (пользовательские параметры)"
|
||||
} | tee -a "$RESULTS/functional.txt"
|
||||
[ "$FAILN" -eq 0 ]
|
||||
78
test/acceptance/lib/helpers.py
Normal file
78
test/acceptance/lib/helpers.py
Normal file
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Small helpers for functional.sh (numpy + rasterio).
|
||||
|
||||
helpers.py ndvi OUT.tif write the 600x400 EPSG:4326 Float32 NDVI test raster
|
||||
helpers.py rinfo FILE.tif print raster facts as JSON (driver, size, crs, dtype,
|
||||
nodata, layout, compression, overviews)
|
||||
helpers.py durations RUN.json print "total=<s> nodes=<min>..<max>ms" for a run
|
||||
helpers.py elapsed DATASET.json print updated_at - created_at of a dataset, seconds
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def ndvi(out):
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from rasterio.transform import from_origin
|
||||
|
||||
w, h = 600, 400 # 0.01 deg pixels, bbox [68.5, 51.5, 74.5, 55.5]
|
||||
y, x = np.mgrid[0:h, 0:w].astype(np.float32)
|
||||
v = 0.45 + 0.35 * np.sin(x / 47.0) * np.cos(y / 31.0) + 0.1 * np.sin((x + y) / 13.0)
|
||||
v = np.clip(v, -0.2, 1.0).astype(np.float32)
|
||||
v[:5, :] = -9999
|
||||
profile = dict(driver="GTiff", width=w, height=h, count=1, dtype="float32", crs="EPSG:4326",
|
||||
transform=from_origin(68.5, 55.5, 0.01, 0.01), nodata=-9999)
|
||||
with rasterio.open(out, "w", **profile) as dst:
|
||||
dst.write(v, 1)
|
||||
|
||||
|
||||
def rinfo(path):
|
||||
import rasterio
|
||||
|
||||
with rasterio.open(path) as d:
|
||||
st = d.tags(ns="IMAGE_STRUCTURE")
|
||||
print(json.dumps({
|
||||
"driver": d.driver, "width": d.width, "height": d.height,
|
||||
"epsg": d.crs.to_epsg() if d.crs else None, "dtype": d.dtypes[0], "nodata": d.nodata,
|
||||
"layout": st.get("LAYOUT"), "compression": st.get("COMPRESSION"),
|
||||
"overviews": len(d.overviews(1)), "bounds": [round(b, 6) for b in d.bounds],
|
||||
}))
|
||||
|
||||
|
||||
def ts(s):
|
||||
if not s:
|
||||
return None
|
||||
s = s.replace("Z", "+00:00")
|
||||
if "." in s: # trim nanoseconds to microseconds for fromisoformat
|
||||
head, rest = s.split(".", 1)
|
||||
frac = "".join(ch for ch in rest if ch.isdigit())
|
||||
tz = rest[len(frac):]
|
||||
s = "%s.%s%s" % (head, frac[:6].ljust(6, "0"), tz)
|
||||
return datetime.fromisoformat(s)
|
||||
|
||||
|
||||
def durations(path):
|
||||
with open(path, encoding="utf-8") as f:
|
||||
run = json.load(f)
|
||||
total = None
|
||||
if run.get("started_at") and run.get("finished_at"):
|
||||
total = (ts(run["finished_at"]) - ts(run["started_at"])).total_seconds()
|
||||
nd = [(ts(n["finished_at"]) - ts(n["started_at"])).total_seconds() * 1000
|
||||
for n in run.get("nodes", []) if n.get("started_at") and n.get("finished_at")]
|
||||
out = "total=%.2fs" % total if total is not None else "total=?"
|
||||
if nd:
|
||||
out += " nodes=%d..%dms" % (min(nd), max(nd))
|
||||
print(out)
|
||||
|
||||
|
||||
def elapsed(path):
|
||||
with open(path, encoding="utf-8") as f:
|
||||
d = json.load(f)
|
||||
print("%.2f" % (ts(d["updated_at"]) - ts(d["created_at"])).total_seconds())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cmd, arg = sys.argv[1], sys.argv[2]
|
||||
{"ndvi": ndvi, "rinfo": rinfo, "durations": durations, "elapsed": elapsed}[cmd](arg)
|
||||
11
test/acceptance/lib/hey.Dockerfile
Normal file
11
test/acceptance/lib/hey.Dockerfile
Normal file
@@ -0,0 +1,11 @@
|
||||
# Load generator image for load.sh: hey v0.1.5 (github.com/rakyll/hey) built
|
||||
# from source on the same pinned Go/Alpine images as the application.
|
||||
FROM golang:1.26.9-alpine3.24 AS build
|
||||
RUN CGO_ENABLED=0 go install github.com/rakyll/hey@v0.1.5
|
||||
FROM alpine:3.24
|
||||
COPY --from=build /go/bin/hey /usr/local/bin/hey
|
||||
RUN adduser -D -u 10001 hey
|
||||
USER hey
|
||||
# One-shot CLI container: nothing to health-check.
|
||||
HEALTHCHECK NONE
|
||||
ENTRYPOINT ["hey"]
|
||||
200
test/acceptance/lib/load_analyze.py
Normal file
200
test/acceptance/lib/load_analyze.py
Normal file
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Aggregates the hey outputs of load.sh into the tables of Приложение 3.
|
||||
|
||||
load_analyze.py RESULTS_DIR
|
||||
|
||||
RESULTS_DIR/manifest.csv rows: scenario,mode,label,clients,file
|
||||
mode par = the files of the scenario ran at the same time (duration = max)
|
||||
seq = they ran one after another (duration = sum)
|
||||
Each file is a default (summary) hey output. Per scenario the program reports
|
||||
requests, clients, duration, RPS, success %, status codes, transport errors,
|
||||
min/avg/P50/P90/P95/P99/max (ms). As in the protocol, percentiles of a
|
||||
scenario made of several hey processes/runs are averages weighted by the
|
||||
number of responses; min/max are exact.
|
||||
|
||||
Criteria (Приложение 3, табл. 8): K1 success >= 99 %, K2 no 5xx, K3 P95 <=
|
||||
2000 ms; K4 (S6 vs S1): /readyz 200 and S6 P50 <= 2 x S1 P50 + 1 ms.
|
||||
|
||||
Writes summary.csv, summary.json, detail.csv into RESULTS_DIR and prints the
|
||||
tables. Exit code 1 when a criterion fails.
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
PCTS = ["10", "25", "50", "75", "90", "95", "99"]
|
||||
|
||||
|
||||
def parse_hey(path):
|
||||
txt = open(path, encoding="utf-8", errors="replace").read()
|
||||
|
||||
def num(label):
|
||||
m = re.search(r"^\s*%s:\s+([0-9.]+)" % re.escape(label), txt, re.M)
|
||||
return float(m.group(1)) if m else None
|
||||
|
||||
r = {"total_s": num("Total"), "slowest_s": num("Slowest"), "fastest_s": num("Fastest"),
|
||||
"average_s": num("Average"), "rps": num("Requests/sec"), "codes": {}, "errors": 0, "pct": {}}
|
||||
for p, v in re.findall(r"^\s*(\d+)%+ in ([0-9.]+) secs", txt, re.M):
|
||||
if p != "0": # hey 0.1.5 prints "%%" and a bogus "0%% in 0.0000" row for tiny samples
|
||||
r["pct"][p] = float(v)
|
||||
sec = txt.split("Status code distribution:", 1)
|
||||
if len(sec) == 2:
|
||||
for code, n in re.findall(r"^\s*\[(\d+)\]\s+(\d+) responses", sec[1].split("Error distribution:")[0], re.M):
|
||||
r["codes"][code] = r["codes"].get(code, 0) + int(n)
|
||||
sec = txt.split("Error distribution:", 1)
|
||||
if len(sec) == 2:
|
||||
r["errors"] = sum(int(n) for n in re.findall(r"^\s*\[(\d+)\]\s", sec[1], re.M))
|
||||
r["responses"] = sum(r["codes"].values())
|
||||
r["requests"] = r["responses"] + r["errors"]
|
||||
return r
|
||||
|
||||
|
||||
def aggregate(rows, mode):
|
||||
parts = [x["hey"] for x in rows]
|
||||
resp = sum(p["responses"] for p in parts)
|
||||
req = sum(p["requests"] for p in parts)
|
||||
durs = [p["total_s"] or 0 for p in parts]
|
||||
dur = max(durs) if mode == "par" else sum(durs)
|
||||
codes = {}
|
||||
for p in parts:
|
||||
for c, n in p["codes"].items():
|
||||
codes[c] = codes.get(c, 0) + n
|
||||
|
||||
def wavg(get):
|
||||
num = sum(get(p) * p["responses"] for p in parts if get(p) is not None and p["responses"])
|
||||
den = sum(p["responses"] for p in parts if get(p) is not None and p["responses"])
|
||||
return num / den if den else None
|
||||
|
||||
ms = lambda v: None if v is None else v * 1000.0
|
||||
ok2xx = sum(n for c, n in codes.items() if c.startswith("2"))
|
||||
fast = [p["fastest_s"] for p in parts if p["fastest_s"] is not None]
|
||||
slow = [p["slowest_s"] for p in parts if p["slowest_s"] is not None]
|
||||
return {
|
||||
"requests": req, "responses": resp, "errors": sum(p["errors"] for p in parts),
|
||||
"clients": sum(int(x["clients"]) for x in rows) if mode == "par" else max(int(x["clients"]) for x in rows),
|
||||
"duration_s": dur, "rps": (resp / dur) if dur else None,
|
||||
"success_pct": (100.0 * ok2xx / req) if req else 0.0, "codes": codes,
|
||||
"n5xx": sum(n for c, n in codes.items() if c.startswith("5")),
|
||||
"min_ms": ms(min(fast)) if fast else None, "avg_ms": ms(wavg(lambda p: p["average_s"])),
|
||||
**{"p%s_ms" % k: ms(wavg(lambda p, k=k: p["pct"].get(k))) for k in ("50", "90", "95", "99")},
|
||||
"max_ms": ms(max(slow)) if slow else None,
|
||||
}
|
||||
|
||||
|
||||
def f1(v, nd=1):
|
||||
return "-" if v is None else ("%.*f" % (nd, v))
|
||||
|
||||
|
||||
def codes_str(c):
|
||||
return ", ".join("%s: %d" % (k, v) for k, v in sorted(c.items())) or "-"
|
||||
|
||||
|
||||
def main():
|
||||
d = sys.argv[1]
|
||||
rows = []
|
||||
with open(os.path.join(d, "manifest.csv"), encoding="utf-8") as f:
|
||||
for x in csv.DictReader(f):
|
||||
x["hey"] = parse_hey(os.path.join(d, "raw", x["file"]))
|
||||
rows.append(x)
|
||||
order, by = [], {}
|
||||
for x in rows:
|
||||
if x["scenario"] not in by:
|
||||
order.append(x["scenario"])
|
||||
by[x["scenario"]] = []
|
||||
by[x["scenario"]].append(x)
|
||||
|
||||
summary = []
|
||||
for s in order:
|
||||
agg = aggregate(by[s], by[s][0]["mode"])
|
||||
agg["scenario"] = s
|
||||
agg["k1"] = agg["success_pct"] >= 99.0
|
||||
agg["k2"] = agg["n5xx"] == 0
|
||||
agg["k3"] = agg["p95_ms"] is not None and agg["p95_ms"] <= 2000.0
|
||||
summary.append(agg)
|
||||
|
||||
extra = {}
|
||||
xp = os.path.join(d, "extra.json")
|
||||
if os.path.exists(xp):
|
||||
extra = json.load(open(xp, encoding="utf-8"))
|
||||
k4 = None
|
||||
s1 = next((x for x in summary if x["scenario"] == "S1"), None)
|
||||
s6 = next((x for x in summary if x["scenario"] == "S6"), None)
|
||||
if s1 and s6:
|
||||
k4 = (extra.get("readyz_after") == 200 and s6["p50_ms"] is not None and s1["p50_ms"] is not None
|
||||
and s6["p50_ms"] <= 2 * s1["p50_ms"] + 1.0)
|
||||
|
||||
hdr = ["Сцен.", "Запр.", "Клиентов", "Длит.,с", "RPS", "Успех,%", "min", "avg", "P50", "P90", "P95", "P99", "max", "Коды", "Ошибки", "Итог"]
|
||||
lines = []
|
||||
for a in summary:
|
||||
ok = a["k1"] and a["k2"] and a["k3"]
|
||||
lines.append([a["scenario"], str(a["requests"]), str(a["clients"]), f1(a["duration_s"]), f1(a["rps"], 0),
|
||||
f1(a["success_pct"], 2), f1(a["min_ms"]), f1(a["avg_ms"]), f1(a["p50_ms"]), f1(a["p90_ms"]),
|
||||
f1(a["p95_ms"]), f1(a["p99_ms"]), f1(a["max_ms"]), codes_str(a["codes"]), str(a["errors"]),
|
||||
"PASS" if ok else "FAIL"])
|
||||
print_table("Сводные результаты (время — мс)", hdr, lines)
|
||||
|
||||
for s in order:
|
||||
if len(by[s]) > 1 and s not in ("S1", "S6"):
|
||||
dl = []
|
||||
for x in by[s]:
|
||||
a = aggregate([x], "par")
|
||||
dl.append([x["label"], x["clients"], str(a["responses"]), f1(a["rps"], 1), f1(a["p50_ms"]),
|
||||
f1(a["p95_ms"]), f1(a["p99_ms"]), f1(a["max_ms"]), codes_str(a["codes"]), str(a["errors"])])
|
||||
print_table("%s по эндпоинтам / прогонам (время — мс)" % s,
|
||||
["Эндпоинт / прогон", "Клиентов", "Ответов", "RPS", "P50", "P95", "P99", "max", "Коды", "Ошибки"], dl)
|
||||
|
||||
if extra.get("uploads"):
|
||||
u = extra["uploads"]
|
||||
print("S5 загрузки: %d запросов POST /datasets, коды %s, время ответа %s–%s мс; обработано до ready: %d/%d, макс. время обработки %s с"
|
||||
% (u["count"], u["codes"], f1(u.get("min_ms")), f1(u.get("max_ms")), u.get("ready", 0), u["count"], f1(u.get("max_processing_s"), 2)))
|
||||
if extra.get("stats_peak"):
|
||||
print("\nПиковое потребление ресурсов (docker stats, сценарии S4/S5):")
|
||||
for name, v in sorted(extra["stats_peak"].items()):
|
||||
print(" %-34s CPU %6.1f %% память %8.1f МиБ" % (name, v["cpu"], v["mem_mib"]))
|
||||
if k4 is not None:
|
||||
print("\nK4 (после нагрузки): /readyz %s; S6 P50/P95 = %s/%s мс против S1 %s/%s мс -> %s" % (
|
||||
extra.get("readyz_after"), f1(s6["p50_ms"]), f1(s6["p95_ms"]), f1(s1["p50_ms"]), f1(s1["p95_ms"]),
|
||||
"PASS" if k4 else "FAIL"))
|
||||
|
||||
with open(os.path.join(d, "summary.csv"), "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["scenario", "requests", "clients", "duration_s", "rps", "success_pct", "min_ms", "avg_ms",
|
||||
"p50_ms", "p90_ms", "p95_ms", "p99_ms", "max_ms", "codes", "errors", "result"])
|
||||
for ln, a in zip(lines, summary):
|
||||
w.writerow([a["scenario"], a["requests"], a["clients"], a["duration_s"], a["rps"], a["success_pct"],
|
||||
a["min_ms"], a["avg_ms"], a["p50_ms"], a["p90_ms"], a["p95_ms"], a["p99_ms"], a["max_ms"],
|
||||
codes_str(a["codes"]), a["errors"], ln[-1]])
|
||||
with open(os.path.join(d, "detail.csv"), "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["scenario", "label", "clients", "requests", "responses", "errors", "duration_s", "rps",
|
||||
"min_ms", "avg_ms", "p50_ms", "p90_ms", "p95_ms", "p99_ms", "max_ms", "codes", "file"])
|
||||
for x in rows:
|
||||
a = aggregate([x], "par")
|
||||
w.writerow([x["scenario"], x["label"], x["clients"], a["requests"], a["responses"], a["errors"],
|
||||
a["duration_s"], a["rps"], a["min_ms"], a["avg_ms"], a["p50_ms"], a["p90_ms"], a["p95_ms"],
|
||||
a["p99_ms"], a["max_ms"], codes_str(a["codes"]), x["file"]])
|
||||
with open(os.path.join(d, "summary.json"), "w", encoding="utf-8") as f:
|
||||
json.dump({"scenarios": summary, "k4": k4, "extra": extra}, f, ensure_ascii=False, indent=2)
|
||||
|
||||
failed = [a["scenario"] for a in summary if not (a["k1"] and a["k2"] and a["k3"])]
|
||||
if k4 is False:
|
||||
failed.append("K4")
|
||||
print("\nКритерии K1 (успех >= 99 %%), K2 (5xx = 0), K3 (P95 <= 2000 мс)%s: %s" % (
|
||||
", K4" if k4 is not None else "", "все выполнены" if not failed else "НЕ выполнены: " + ", ".join(failed)))
|
||||
print("Файлы: %s/{summary.csv,summary.json,detail.csv,raw/}" % d)
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def print_table(title, hdr, lines):
|
||||
widths = [max(len(str(r[i])) for r in [hdr] + lines) for i in range(len(hdr))]
|
||||
print("\n" + title)
|
||||
print(" | ".join(h.ljust(w) for h, w in zip(hdr, widths)))
|
||||
print("-+-".join("-" * w for w in widths))
|
||||
for r in lines:
|
||||
print(" | ".join(str(c).ljust(w) for c, w in zip(r, widths)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
150
test/acceptance/lib/make_fixtures.py
Normal file
150
test/acceptance/lib/make_fixtures.py
Normal file
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regenerates the committed vector/SLD fixtures in test/acceptance/data/.
|
||||
|
||||
The fixtures are committed, so this script is only needed to rebuild them.
|
||||
Requires the GDAL command-line tool ogr2ogr (GeoPackage conversion).
|
||||
|
||||
python3 test/acceptance/lib/make_fixtures.py
|
||||
|
||||
Values are deterministic: district KATO codes and names come from
|
||||
stubs/districts.geojson (14 districts of North Kazakhstan region), the
|
||||
indicator values follow a fixed formula, so re-running yields identical files.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.abspath(os.path.join(HERE, "..", "..", ".."))
|
||||
DATA = os.path.abspath(os.path.join(HERE, "..", "data"))
|
||||
|
||||
|
||||
def ring_bbox(coords):
|
||||
xs, ys = [], []
|
||||
|
||||
def walk(c):
|
||||
if isinstance(c[0], (int, float)):
|
||||
xs.append(c[0])
|
||||
ys.append(c[1])
|
||||
else:
|
||||
for x in c:
|
||||
walk(x)
|
||||
|
||||
walk(coords)
|
||||
return min(xs), min(ys), max(xs), max(ys)
|
||||
|
||||
|
||||
def population(i, kato, year):
|
||||
# Values quoted in the protocol (Приложение 7, F16/F17) are kept verbatim.
|
||||
fixed = {
|
||||
("591000000", 2022): 47567.6,
|
||||
("591000000", 2023): 48311.2,
|
||||
("591000000", 2024): 49120.5,
|
||||
("595000000", 2022): 46862.7,
|
||||
("595000000", 2023): 48268.6,
|
||||
("595000000", 2024): 49205.8,
|
||||
}
|
||||
if (kato, year) in fixed:
|
||||
return fixed[(kato, year)]
|
||||
return round(20000 + 1750.3 * i + 812.4 * (year - 2022) + (i * 37 % 11) * 13.1, 1)
|
||||
|
||||
|
||||
def livestock(i, year):
|
||||
return int(30000 + 2311 * i + 977 * (year - 2022) + (i * 53 % 17) * 41)
|
||||
|
||||
|
||||
def main():
|
||||
with open(os.path.join(ROOT, "stubs", "districts.geojson"), encoding="utf-8") as f:
|
||||
districts = json.load(f)["features"]
|
||||
os.makedirs(DATA, exist_ok=True)
|
||||
|
||||
# 1. plain vector: 4 plot polygons spanning lon 69.0..71.7, lat 53.6..55.0
|
||||
plots = []
|
||||
boxes = [
|
||||
(69.0, 53.6, 69.6, 54.0),
|
||||
(69.9, 54.2, 70.5, 54.7),
|
||||
(70.6, 53.7, 71.1, 54.1),
|
||||
(71.2, 54.5, 71.7, 55.0),
|
||||
]
|
||||
for n, (x0, y0, x1, y1) in enumerate(boxes, 1):
|
||||
plots.append({
|
||||
"type": "Feature",
|
||||
"properties": {"plot_id": n, "name": "Участок %d" % n, "area_ha": round((x1 - x0) * (y1 - y0) * 7000, 1)},
|
||||
"geometry": {"type": "Polygon", "coordinates": [[[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]]]},
|
||||
})
|
||||
write_json("plots_nko.geojson", {"type": "FeatureCollection", "features": plots})
|
||||
|
||||
# 2. vector_with_kato (GeoJSON): population per district, Y2022..Y2024
|
||||
pop = []
|
||||
live = []
|
||||
for i, d in enumerate(districts):
|
||||
p = d["properties"]
|
||||
kato = str(p["kato"])
|
||||
x0, y0, x1, y1 = ring_bbox(d["geometry"]["coordinates"])
|
||||
pt = {"type": "Point", "coordinates": [round((x0 + x1) / 2, 5), round((y0 + y1) / 2, 5)]}
|
||||
props = {"kato": kato, "district": p["name_ru"]}
|
||||
for y in (2022, 2023, 2024):
|
||||
props["Y%d" % y] = population(i, kato, y)
|
||||
pop.append({"type": "Feature", "properties": props, "geometry": pt})
|
||||
lprops = {"KATO_CODE": kato, "district": p["name_ru"]}
|
||||
for y in (2022, 2023, 2024):
|
||||
lprops["H_%d" % y] = livestock(i, y)
|
||||
live.append({"type": "Feature", "properties": lprops, "geometry": pt})
|
||||
write_json("population_kato.geojson", {"type": "FeatureCollection", "features": pop})
|
||||
|
||||
# 3. vector_with_kato (GeoPackage): cattle per district, H_2022..H_2024
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
src = os.path.join(tmp, "livestock.geojson")
|
||||
with open(src, "w", encoding="utf-8") as f:
|
||||
json.dump({"type": "FeatureCollection", "features": live}, f, ensure_ascii=False)
|
||||
dst = os.path.join(DATA, "livestock_kato.gpkg")
|
||||
if os.path.exists(dst):
|
||||
os.remove(dst)
|
||||
subprocess.run(["ogr2ogr", "-f", "GPKG", "-nln", "livestock", "-a_srs", "EPSG:4326", dst, src], check=True)
|
||||
|
||||
# 4. SLD styles
|
||||
write_text("style.sld", sld("ndvi", "NDVI gradient", [("0.0", "#a50026"), ("0.5", "#ffffbf"), ("1.0", "#006837")]))
|
||||
write_text("group_style.sld", sld("group", "Group fallback style", [("0", "#f7fbff"), ("100000", "#08306b")]))
|
||||
write_text("style.txt", "not an SLD file\n")
|
||||
print("fixtures written to", DATA)
|
||||
|
||||
|
||||
def sld(name, title, stops):
|
||||
entries = "\n".join(
|
||||
' <sld:ColorMapEntry color="%s" quantity="%s"/>' % (c, q) for q, c in stops)
|
||||
return """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<sld:StyledLayerDescriptor version="1.0.0" xmlns:sld="http://www.opengis.net/sld" xmlns:ogc="http://www.opengis.net/ogc">
|
||||
<sld:NamedLayer>
|
||||
<sld:Name>%s</sld:Name>
|
||||
<sld:UserStyle>
|
||||
<sld:Title>%s</sld:Title>
|
||||
<sld:FeatureTypeStyle>
|
||||
<sld:Rule>
|
||||
<sld:RasterSymbolizer>
|
||||
<sld:ColorMap>
|
||||
%s
|
||||
</sld:ColorMap>
|
||||
</sld:RasterSymbolizer>
|
||||
</sld:Rule>
|
||||
</sld:FeatureTypeStyle>
|
||||
</sld:UserStyle>
|
||||
</sld:NamedLayer>
|
||||
</sld:StyledLayerDescriptor>
|
||||
""" % (name, title, entries)
|
||||
|
||||
|
||||
def write_json(name, obj):
|
||||
with open(os.path.join(DATA, name), "w", encoding="utf-8") as f:
|
||||
json.dump(obj, f, ensure_ascii=False, indent=1)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def write_text(name, text):
|
||||
with open(os.path.join(DATA, name), "w", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
227
test/acceptance/load.sh
Executable file
227
test/acceptance/load.sh
Executable file
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env bash
|
||||
# Load test of the GIS API — scenarios S1–S6 of «Протокол нагрузочного
|
||||
# тестирования» (Приложение 3), generator hey 0.1.5:
|
||||
# S1 baseline: 10 endpoints one after another, hey -n 100 -c 1 each
|
||||
# S2 mixed: 1000 requests, 100 concurrent clients over the 10 endpoints
|
||||
# (10 parallel hey processes, -n 100 -c 10 each)
|
||||
# S3.1 1000 simultaneous requests: hey -n 1000 -c 1000 /datasets?page_size=20, 3 runs
|
||||
# S3.2 1000 simultaneous requests: hey -n 1000 -c 1000 /landscape/results, 3 runs
|
||||
# S4 sustained ~1000 req/s for 60 s: 1000 clients at -q 1 split over 8
|
||||
# endpoints (300/150/100/100/100/100/100/50), 8 parallel hey processes
|
||||
# S5 30 s of reads (10 endpoints x 10 clients, -z 30s) while 20 uploads
|
||||
# (10 GeoJSON + 10 GeoTIFF) run in parallel; uploads must reach ready
|
||||
# S6 recovery: S1 again + /readyz
|
||||
#
|
||||
# By default the generator runs INSIDE the stand's docker network
|
||||
# (docker run --network gis-acceptance_default gis-acceptance-hey:0.1.5,
|
||||
# target http://app:8080), so host port forwarding is not measured.
|
||||
#
|
||||
# Environment:
|
||||
# BASE_URL API address for setup calls (default http://localhost:18080)
|
||||
# TARGET_URL address the generator loads (default http://app:8080 inside
|
||||
# the network; any URL with GEN=host or NETWORK=bridge)
|
||||
# GEN docker (default) | host (use a locally installed hey)
|
||||
# NETWORK docker network of the generator (default gis-acceptance_default)
|
||||
# SCENARIOS subset to run (default "S1 S2 S3 S4 S5 S6")
|
||||
# S4_DURATION / S5_DURATION default 60s / 30s
|
||||
# PYTHON python3 with numpy + rasterio (S5 raster uploads, seeding)
|
||||
# ALLOW_WRITES 1 = allow seeding and S5 uploads on a non-local BASE_URL
|
||||
#
|
||||
# Output: summary table on stdout; test/acceptance/results/load-<timestamp>/
|
||||
# {summary.csv, summary.json, detail.csv, raw/*.txt}. Exit code 1 when a
|
||||
# criterion K1–K4 fails.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
BASE_URL="${BASE_URL:-http://localhost:18080}"; BASE_URL="${BASE_URL%/}"
|
||||
GEN="${GEN:-docker}"
|
||||
NETWORK="${NETWORK:-gis-acceptance_default}"
|
||||
HEY_IMAGE="${HEY_IMAGE:-gis-acceptance-hey:0.1.5}"
|
||||
SCENARIOS="${SCENARIOS:-S1 S2 S3 S4 S5 S6}"
|
||||
S4_DURATION="${S4_DURATION:-60s}"
|
||||
S5_DURATION="${S5_DURATION:-30s}"
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
ALLOW_WRITES="${ALLOW_WRITES:-0}"
|
||||
if [ -z "${TARGET_URL:-}" ]; then
|
||||
if [ "$GEN" = docker ] && [ "$NETWORK" = gis-acceptance_default ]; then TARGET_URL=http://app:8080; else TARGET_URL="$BASE_URL"; fi
|
||||
fi
|
||||
TARGET_URL="${TARGET_URL%/}"
|
||||
LOCAL=0
|
||||
case "$BASE_URL" in http://localhost:*|http://127.0.0.1:*) LOCAL=1;; esac
|
||||
case "$BASE_URL" in *dssgis.dwh.kz*) ALLOW_WRITES=0;; esac
|
||||
|
||||
for bin in curl jq python3; do command -v "$bin" >/dev/null || { echo "missing $bin" >&2; exit 2; }; done
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
OUT="$HERE/results/load-$STAMP"; RAW="$OUT/raw"; mkdir -p "$RAW"
|
||||
echo "scenario,mode,label,clients,file" > "$OUT/manifest.csv"
|
||||
EXTRA="$OUT/extra.json"; echo '{}' > "$EXTRA"
|
||||
log() { echo "[$(date +%H:%M:%S)] $*"; }
|
||||
extra_set() { local t; t="$(mktemp)"; jq "$1" "$EXTRA" > "$t" && mv "$t" "$EXTRA"; }
|
||||
|
||||
# ---------------------------------------------------------------- generator
|
||||
if [ "$GEN" = docker ]; then
|
||||
if ! docker image inspect "$HEY_IMAGE" >/dev/null 2>&1; then
|
||||
log "building $HEY_IMAGE"
|
||||
docker build -q -t "$HEY_IMAGE" -f "$HERE/lib/hey.Dockerfile" "$HERE/lib" >/dev/null
|
||||
fi
|
||||
NETARG="--network $NETWORK"; [ "$NETWORK" = bridge ] && NETARG=""
|
||||
fi
|
||||
# run_batch SCEN MODE "label|clients|file|hey args and path" ...
|
||||
# MODE par: all hey processes start together; seq: one after another.
|
||||
run_batch() {
|
||||
local scen="$1" mode="$2"; shift 2
|
||||
local script="" spec label clients file args sep=" & "
|
||||
[ "$mode" = seq ] && sep=" ; "
|
||||
for spec in "$@"; do
|
||||
IFS='|' read -r label clients file args <<< "$spec"
|
||||
echo "$scen,$mode,\"$label\",$clients,$file" >> "$OUT/manifest.csv"
|
||||
script="${script}hey $args > /out/$file 2>&1${sep}"
|
||||
done
|
||||
script="${script} wait"
|
||||
if [ "$GEN" = docker ]; then
|
||||
# shellcheck disable=SC2086
|
||||
docker run --rm $NETARG --ulimit nofile=65536:65536 -v "$RAW:/out" --entrypoint sh "$HEY_IMAGE" -c "$script"
|
||||
else
|
||||
( cd "$RAW" && ulimit -n 65536 2>/dev/null || true; sh -c "${script//\/out\//$RAW/}" )
|
||||
fi
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------- endpoints
|
||||
if [ "$LOCAL" = 1 ] || [ "$ALLOW_WRITES" = 1 ]; then
|
||||
log "seed (idempotent): $BASE_URL"
|
||||
BASE_URL="$BASE_URL" PYTHON="$PYTHON" "$HERE/seed.sh" | tail -n 4
|
||||
# shellcheck disable=SC1091
|
||||
. "$HERE/results/seed.env"
|
||||
else
|
||||
log "read-only discovery of endpoint ids on $BASE_URL"
|
||||
VWK_ID="$(curl -fsS "$BASE_URL/datasets?file_type=vector_with_kato&status=ready&page_size=1" | jq -r '.data[0].id')"
|
||||
GROUP_ID="$(curl -fsS "$BASE_URL/datasets/$VWK_ID" | jq -r '.dg_id // empty')"
|
||||
CAT_ID=""
|
||||
fi
|
||||
RUN_ID="$(curl -fsS "$BASE_URL/landscape/runs?page_size=1" | jq -r '.data[0].id // empty')"
|
||||
if [ -z "$RUN_ID" ] && { [ "$LOCAL" = 1 ] || [ "$ALLOW_WRITES" = 1 ]; }; then
|
||||
log "no ЛП run yet: running lp_run.sh"
|
||||
BASE_URL="$BASE_URL" PYTHON="$PYTHON" "$HERE/lp_run.sh" > "$OUT/lp_run.txt" 2>&1
|
||||
RUN_ID="$(curl -fsS "$BASE_URL/landscape/runs?page_size=1" | jq -r '.data[0].id')"
|
||||
fi
|
||||
for v in VWK_ID GROUP_ID RUN_ID; do
|
||||
eval "val=\${$v:-}"
|
||||
# shellcheck disable=SC2154
|
||||
[ -n "$val" ] && [ "$val" != null ] || { echo "cannot resolve $v on $BASE_URL" >&2; exit 2; }
|
||||
done
|
||||
EP_PATHS="/datasets?page_size=20 /datasets/$VWK_ID /datasets/$VWK_ID.geojson /datasets/$VWK_ID/observations /categories /dataset-groups/$GROUP_ID/latest /landscape/slots /landscape/results /landscape/runs/$RUN_ID /readyz"
|
||||
label_of() { # path -> protocol label
|
||||
echo "$1" | sed -e "s#$VWK_ID#{id}#g" -e "s#$GROUP_ID#{id}#g" -e "s#$RUN_ID#{id}#g"
|
||||
}
|
||||
fname() { echo "$1" | sed -e 's#^/##' -e 's#[/?=.{}]#_#g'; }
|
||||
for p in $EP_PATHS; do # every endpoint must answer 200 before loading it
|
||||
c="$(curl -sS -o /dev/null -w '%{http_code}' "$BASE_URL$p")"
|
||||
[ "$c" = 200 ] || { echo "endpoint $p answers $c, fix the stand first" >&2; exit 2; }
|
||||
done
|
||||
log "target $TARGET_URL (generator: $GEN${NETARG:+, $NETARG}); results $OUT"
|
||||
curl -fsS "$BASE_URL/datasets?status=ready&page_size=1" | jq -r '"ready datasets on the stand: \(.total)"'
|
||||
|
||||
stats_start() { # docker stats sampler for the compose project (local stand only)
|
||||
[ "$LOCAL" = 1 ] || return 0
|
||||
( while :; do docker stats --no-stream --format '{{.Name}},{{.CPUPerc}},{{.MemUsage}}' \
|
||||
$(docker ps -q --filter label=com.docker.compose.project=gis-acceptance) >> "$OUT/stats.csv" 2>/dev/null || true
|
||||
sleep 3; done ) &
|
||||
STATS_PID=$!
|
||||
}
|
||||
stats_stop() { [ -n "${STATS_PID:-}" ] && kill "$STATS_PID" 2>/dev/null || true; STATS_PID=""; }
|
||||
trap 'stats_stop' EXIT
|
||||
|
||||
baseline() { # S1 / S6
|
||||
local scen="$1" specs=() p
|
||||
for p in $EP_PATHS; do specs+=("$(label_of "$p")|1|$scen-$(fname "$p").txt|-n 100 -c 1 '$TARGET_URL$p'"); done
|
||||
log "$scen: 10 endpoints x hey -n 100 -c 1 (sequential)"
|
||||
run_batch "$scen" seq "${specs[@]}"
|
||||
}
|
||||
|
||||
for S in $SCENARIOS; do
|
||||
case "$S" in
|
||||
S1) baseline S1 ;;
|
||||
S2)
|
||||
specs=()
|
||||
for p in $EP_PATHS; do specs+=("$(label_of "$p")|10|S2-$(fname "$p").txt|-n 100 -c 10 '$TARGET_URL$p'"); done
|
||||
log "S2: 1000 requests, 100 clients over 10 endpoints"
|
||||
run_batch S2 par "${specs[@]}" ;;
|
||||
S3)
|
||||
for i in 1 2 3; do
|
||||
log "S3.1 run $i: hey -n 1000 -c 1000 /datasets?page_size=20"
|
||||
run_batch S3.1 seq "/datasets?page_size=20 №$i|1000|S3.1-run$i.txt|-n 1000 -c 1000 '$TARGET_URL/datasets?page_size=20'"
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
log "S3.2 run $i: hey -n 1000 -c 1000 /landscape/results"
|
||||
run_batch S3.2 seq "/landscape/results №$i|1000|S3.2-run$i.txt|-n 1000 -c 1000 '$TARGET_URL/landscape/results'"
|
||||
done ;;
|
||||
S4)
|
||||
specs=()
|
||||
for pair in "/datasets?page_size=20:300" "/datasets/$VWK_ID:150" "/datasets/$VWK_ID.geojson:100" "/datasets/$VWK_ID/observations:100" \
|
||||
"/categories:100" "/dataset-groups/$GROUP_ID/latest:100" "/landscape/results:100" "/readyz:50"; do
|
||||
p="${pair%:*}"; c="${pair##*:}"
|
||||
specs+=("$(label_of "$p")|$c|S4-$(fname "$p").txt|-z $S4_DURATION -q 1 -c $c '$TARGET_URL$p'")
|
||||
done
|
||||
log "S4: 1000 clients at 1 req/s each for $S4_DURATION (8 hey processes)"
|
||||
stats_start; run_batch S4 par "${specs[@]}"; stats_stop ;;
|
||||
S5)
|
||||
if [ "$LOCAL" != 1 ] && [ "$ALLOW_WRITES" != 1 ]; then log "S5 skipped: uploads need a local stand or ALLOW_WRITES=1"; continue; fi
|
||||
TMP5="$(mktemp -d "${TMPDIR:-/tmp}/gis-load.XXXXXX")"
|
||||
"$PYTHON" "$HERE/lib/helpers.py" ndvi "$TMP5/ndvi_s5.tif"
|
||||
specs=()
|
||||
for p in $EP_PATHS; do specs+=("$(label_of "$p")|10|S5-$(fname "$p").txt|-z $S5_DURATION -c 10 '$TARGET_URL$p'"); done
|
||||
log "S5: reads 10 x 10 clients for $S5_DURATION + 20 parallel uploads"
|
||||
stats_start
|
||||
run_batch S5 par "${specs[@]}" &
|
||||
READ_PID=$!
|
||||
sleep 5
|
||||
for i in $(seq 1 20); do
|
||||
if [ "$i" -le 10 ]; then f="$HERE/data/plots_nko.geojson"; t=vector; else f="$TMP5/ndvi_s5.tif"; t=raster; fi
|
||||
( curl -sS -o "$TMP5/up$i.json" -w '%{http_code} %{time_total}\n' -X POST "$BASE_URL/datasets" \
|
||||
-F "file=@$f" -F "file_type=$t" -F "category_id=$CAT_ID" -F "name=S5 upload $i" > "$TMP5/up$i.code"; date +%s > "$TMP5/up$i.t" ) &
|
||||
done
|
||||
for i in $(seq 1 20); do while [ ! -s "$TMP5/up$i.t" ]; do sleep 0.2; done; done
|
||||
wait "$READ_PID" || true
|
||||
stats_stop
|
||||
# processing: every upload must reach ready
|
||||
codes=""; ready=0; maxproc=0; mins=999999; maxs=0
|
||||
for i in $(seq 1 20); do
|
||||
read -r code tt < "$TMP5/up$i.code"; codes="$codes $code"
|
||||
ms="$(awk -v t="$tt" 'BEGIN{printf "%.1f", t*1000}')"
|
||||
mins="$(awk -v a="$mins" -v b="$ms" 'BEGIN{print (b<a)?b:a}')"; maxs="$(awk -v a="$maxs" -v b="$ms" 'BEGIN{print (b>a)?b:a}')"
|
||||
id="$(jq -r '.id // empty' "$TMP5/up$i.json")"; [ -n "$id" ] || continue
|
||||
for _ in $(seq 1 300); do
|
||||
st="$(curl -fsS "$BASE_URL/datasets/$id" | jq -r .status)"
|
||||
[ "$st" = ready ] || [ "$st" = failed ] && break; sleep 1
|
||||
done
|
||||
if [ "$st" = ready ]; then
|
||||
ready=$((ready + 1))
|
||||
# server-side processing time: updated_at (ready) - created_at (upload)
|
||||
curl -fsS "$BASE_URL/datasets/$id" -o "$TMP5/ds$i.json"
|
||||
proc="$(python3 "$HERE/lib/helpers.py" elapsed "$TMP5/ds$i.json" 2>/dev/null || echo 0)"
|
||||
maxproc="$(awk -v a="$maxproc" -v b="$proc" 'BEGIN{print (b>a)?b:a}')"
|
||||
fi
|
||||
curl -sS -o /dev/null -X DELETE "$BASE_URL/datasets/$id"
|
||||
done
|
||||
cs="$(echo $codes | tr ' ' '\n' | sort | uniq -c | awk '{printf "%s%s: %s", (NR>1?", ":""), $2, $1}')"
|
||||
extra_set ".uploads = {count: 20, codes: \"$cs\", min_ms: $mins, max_ms: $maxs, ready: $ready, max_processing_s: $maxproc}"
|
||||
log "S5 uploads: $cs; ready $ready/20 (max processing ${maxproc}s = updated_at - created_at); uploads deleted"
|
||||
rm -rf "$TMP5" ;;
|
||||
S6)
|
||||
baseline S6
|
||||
rz="$(curl -sS -o "$OUT/readyz_after.json" -w '%{http_code}' "$BASE_URL/readyz")"
|
||||
extra_set ".readyz_after = $rz | .readyz_body = $(cat "$OUT/readyz_after.json")"
|
||||
log "S6: /readyz $rz $(cat "$OUT/readyz_after.json")" ;;
|
||||
*) echo "unknown scenario $S" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -s "$OUT/stats.csv" ]; then # peak CPU % / memory MiB per container
|
||||
peaks="$(awk -F, '{gsub("%","",$2); split($3,m," / "); v=m[1]; mult=1;
|
||||
if (v ~ /GiB$/) {mult=1024} else if (v ~ /KiB$/) {mult=1/1024} else if (v ~ /kB$/) {mult=1/1024} else if (v ~ /MB$/) {mult=1};
|
||||
gsub(/[A-Za-z]+$/,"",v); mem=v*mult; if ($2>cpu[$1]) cpu[$1]=$2; if (mem>mm[$1]) mm[$1]=mem }
|
||||
END { printf "{"; n=0; for (k in cpu) { printf "%s\"%s\":{\"cpu\":%s,\"mem_mib\":%.1f}", (n++?",":""), k, cpu[k], mm[k] } printf "}" }' "$OUT/stats.csv")"
|
||||
extra_set ".stats_peak = $peaks"
|
||||
fi
|
||||
|
||||
python3 "$HERE/lib/load_analyze.py" "$OUT"
|
||||
111
test/acceptance/lp_gen.py
Executable file
111
test/acceptance/lp_gen.py
Executable file
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generates the synthetic input rasters of the Landscape Program (ЛП) test.
|
||||
|
||||
Writes 13 grid-aligned GeoTIFFs, one per input slot (file name = slot code),
|
||||
plus one deliberately misaligned raster for the negative test (L13):
|
||||
|
||||
<out>/lp-z-biodiversity.tif ... <out>/lp-p-tourism.tif (13 files)
|
||||
<out>/misaligned-lp-u-veg.tif (half size, 2x pixel)
|
||||
|
||||
Every input raster: int16, EPSG:32642 (UTM 42N), 100 m pixels, default
|
||||
3000 x 2000 pixels, classes 1..5 from smooth trigonometric patterns (a
|
||||
different pattern per slot, so ties and every class combination occur), and a
|
||||
NoData (-9999) frame 40 pixels wide. Extra edge cases exercise the pixel rules:
|
||||
|
||||
* lp-u-crop: a 200 x 200 NoData block — U is computed from water, soil and
|
||||
veg only (the crop/veg union rule of u_integrated / u_max);
|
||||
* lp-u-veg: a 100 x 100 NoData block overlapping half of the crop block —
|
||||
where both crop and veg are missing, U must be NoData;
|
||||
* lp-p-snow: a 120 x 120 block of 0 and a 60 x 60 block of 6 — values that
|
||||
are not NoData but outside 1..5, hence invalid (the "valid iff 1 <= v <= 5"
|
||||
rule).
|
||||
|
||||
Output is deterministic (no random numbers), so the same command always
|
||||
produces byte-identical pixel values.
|
||||
|
||||
python lp_gen.py --out /tmp/lp-inputs [--width 3000 --height 2000]
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from rasterio.transform import from_origin
|
||||
|
||||
SLOTS = [
|
||||
"lp-z-biodiversity", "lp-z-lcr", "lp-z-soil-protection", "lp-z-wr",
|
||||
"lp-u-surf-water", "lp-u-soil", "lp-u-crop", "lp-u-veg",
|
||||
"lp-p-irrigation", "lp-p-snow", "lp-p-crop", "lp-p-pasture", "lp-p-tourism",
|
||||
]
|
||||
NODATA = -9999
|
||||
CRS = "EPSG:32642"
|
||||
ORIGIN_X, ORIGIN_Y = 300000.0, 6100000.0 # UTM 42N, North Kazakhstan region
|
||||
PIXEL = 100.0
|
||||
FRAME = 40
|
||||
|
||||
|
||||
def pattern(k, width, height):
|
||||
"""Smooth class field 1..5 for slot index k."""
|
||||
y, x = np.mgrid[0:height, 0:width].astype(np.float64)
|
||||
fx = 2 * np.pi * (1.0 + 0.37 * k) / width
|
||||
fy = 2 * np.pi * (0.8 + 0.29 * ((k * 5) % 7)) / height
|
||||
v = (np.sin(fx * x + 0.9 * k) * np.cos(fy * y + 0.4 * k)
|
||||
+ 0.5 * np.sin(fx * 2.3 * x + fy * 1.7 * y + k))
|
||||
v = (v + 1.5) / 3.0 # ~0..1
|
||||
cls = np.floor(np.clip(v, 0, 0.999999) * 5).astype(np.int16) + 1
|
||||
return cls
|
||||
|
||||
|
||||
def block(arr, r0, c0, h, w, value):
|
||||
arr[r0:r0 + h, c0:c0 + w] = value
|
||||
|
||||
|
||||
def write(path, arr, transform):
|
||||
profile = dict(driver="GTiff", width=arr.shape[1], height=arr.shape[0], count=1,
|
||||
dtype="int16", crs=CRS, transform=transform, nodata=NODATA,
|
||||
tiled=True, blockxsize=256, blockysize=256, compress="DEFLATE")
|
||||
with rasterio.open(path, "w", **profile) as dst:
|
||||
dst.write(arr, 1)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--out", required=True, help="output directory")
|
||||
ap.add_argument("--width", type=int, default=3000)
|
||||
ap.add_argument("--height", type=int, default=2000)
|
||||
args = ap.parse_args()
|
||||
w, h = args.width, args.height
|
||||
if w < 400 or h < 400:
|
||||
sys.exit("width/height must be >= 400")
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
transform = from_origin(ORIGIN_X, ORIGIN_Y, PIXEL, PIXEL)
|
||||
|
||||
for k, slot in enumerate(SLOTS):
|
||||
arr = pattern(k, w, h)
|
||||
arr[:FRAME, :] = NODATA
|
||||
arr[-FRAME:, :] = NODATA
|
||||
arr[:, :FRAME] = NODATA
|
||||
arr[:, -FRAME:] = NODATA
|
||||
cr, cc = h // 2 - 100, w // 2 - 100 # centre block anchor
|
||||
if slot == "lp-u-crop":
|
||||
block(arr, cr, cc, 200, 200, NODATA)
|
||||
if slot == "lp-u-veg":
|
||||
block(arr, cr + 50, cc + 150, 100, 100, NODATA)
|
||||
if slot == "lp-p-snow":
|
||||
block(arr, 300, 300, 120, 120, 0)
|
||||
block(arr, 300, 500, 60, 60, 6)
|
||||
write(os.path.join(args.out, slot + ".tif"), arr, transform)
|
||||
print("wrote", slot + ".tif")
|
||||
|
||||
# Misaligned raster for L13: half the size, double pixel => same footprint,
|
||||
# different grid.
|
||||
mis = pattern(7, w // 2, h // 2)
|
||||
write(os.path.join(args.out, "misaligned-lp-u-veg.tif"), mis,
|
||||
from_origin(ORIGIN_X, ORIGIN_Y, PIXEL * 2, PIXEL * 2))
|
||||
print("wrote misaligned-lp-u-veg.tif (%dx%d, %d m)" % (w // 2, h // 2, int(PIXEL * 2)))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
72
test/acceptance/lp_run.sh
Executable file
72
test/acceptance/lp_run.sh
Executable file
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
# Landscape Program (ЛП) end-to-end check on the acceptance stand:
|
||||
# 1. lp_gen.py — 13 synthetic input rasters (default 3000 x 2000)
|
||||
# 2. upload — POST /landscape/slots/{code} for the 13 slots
|
||||
# 3. run — POST /landscape/runs {"from_stage":0}, wait for it
|
||||
# 4. lp_verify.py — pixel-by-pixel check of all 15 outputs (mode a)
|
||||
#
|
||||
# test/acceptance/lp_run.sh # BASE_URL defaults to http://localhost:18080
|
||||
# LP_WIDTH=6000 LP_HEIGHT=4000 test/acceptance/lp_run.sh
|
||||
#
|
||||
# The uploaded inputs and the run's outputs are left in place so the run can be
|
||||
# re-verified later (python lp_verify.py --run <id>); they are ordinary datasets
|
||||
# of the "landscape" category. WRITES to the stand: never point it at production.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
BASE_URL="${BASE_URL:-http://localhost:18080}"
|
||||
BASE_URL="${BASE_URL%/}"
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
WAIT_TIMEOUT="${WAIT_TIMEOUT:-600}"
|
||||
LP_WIDTH="${LP_WIDTH:-3000}"
|
||||
LP_HEIGHT="${LP_HEIGHT:-2000}"
|
||||
case "$BASE_URL" in
|
||||
*dssgis.dwh.kz*) echo "refusing to write to $BASE_URL (production); use lp_verify.py for read-only checks" >&2; exit 2;;
|
||||
esac
|
||||
|
||||
TMP="$(mktemp -d "${TMPDIR:-/tmp}/gis-lp.XXXXXX")"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
SLOTS="lp-z-biodiversity lp-z-lcr lp-z-soil-protection lp-z-wr lp-u-surf-water lp-u-soil lp-u-crop lp-u-veg lp-p-irrigation lp-p-snow lp-p-crop lp-p-pasture lp-p-tourism"
|
||||
|
||||
echo "== 1. generate inputs (${LP_WIDTH}x${LP_HEIGHT})"
|
||||
"$PYTHON" "$HERE/lp_gen.py" --out "$TMP/in" --width "$LP_WIDTH" --height "$LP_HEIGHT"
|
||||
|
||||
echo "== 2. upload into 13 slots ($BASE_URL)"
|
||||
ids=""
|
||||
for s in $SLOTS; do
|
||||
out="$(curl -sS -w ' %{http_code}' -X POST "$BASE_URL/landscape/slots/$s" -F "file=@$TMP/in/$s.tif" \
|
||||
-F "name=ЛП приёмка $s $STAMP" -F start_date=2025-01-01 -F end_date=2025-12-31)"
|
||||
code="${out##* }"; body="${out% *}"
|
||||
[ "$code" = 201 ] || { echo "upload $s failed: HTTP $code $body" >&2; exit 1; }
|
||||
id="$(jq -r .id <<< "$body")"; ids="$ids $id"
|
||||
printf ' %-22s %s HTTP %s\n' "$s" "$id" "$code"
|
||||
done
|
||||
for id in $ids; do
|
||||
deadline=$(( $(date +%s) + WAIT_TIMEOUT ))
|
||||
until [ "$(curl -sS "$BASE_URL/datasets/$id" | jq -r .status)" = ready ]; do
|
||||
[ "$(date +%s)" -lt "$deadline" ] || { echo "dataset $id not ready" >&2; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
echo " all 13 inputs ready"
|
||||
|
||||
echo "== 3. run {\"from_stage\":0}"
|
||||
run="$(curl -sS -X POST "$BASE_URL/landscape/runs" -H 'Content-Type: application/json' -d '{"from_stage":0}')"
|
||||
RUN="$(jq -r .id <<< "$run")"
|
||||
[ -n "$RUN" ] && [ "$RUN" != null ] || { echo "run not created: $run" >&2; exit 1; }
|
||||
echo " run $RUN queued"
|
||||
deadline=$(( $(date +%s) + WAIT_TIMEOUT ))
|
||||
while :; do
|
||||
st="$(curl -sS "$BASE_URL/landscape/runs/$RUN" | jq -r .status)"
|
||||
case "$st" in succeeded|failed) break;; esac
|
||||
[ "$(date +%s)" -lt "$deadline" ] || { echo "run $RUN still $st" >&2; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
curl -sS "$BASE_URL/landscape/runs/$RUN" > "$TMP/run.json"
|
||||
echo " run $RUN $st: $(jq -r '[.nodes[] | "\(.node_code):\(.status)"] | sort | join(" ")' "$TMP/run.json")"
|
||||
"$PYTHON" "$HERE/lib/helpers.py" durations "$TMP/run.json" | sed 's/^/ /'
|
||||
[ "$st" = succeeded ] || { echo "run failed: $(jq -r .error "$TMP/run.json")" >&2; exit 1; }
|
||||
|
||||
echo "== 4. lp_verify.py --base-url $BASE_URL --run $RUN"
|
||||
"$PYTHON" "$HERE/lp_verify.py" --base-url "$BASE_URL" --run "$RUN" --cache "$TMP/cache"
|
||||
436
test/acceptance/lp_verify.py
Executable file
436
test/acceptance/lp_verify.py
Executable file
@@ -0,0 +1,436 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent pixel-by-pixel check of a Landscape Program (ЛП) run.
|
||||
|
||||
Re-implements every node of the ЛП graph in numpy — 00 z_integrated,
|
||||
01 u_integrated, 02 z_max, 03 u_max, 10 e_ecological_constraint,
|
||||
20 c_conflicts, 21 p_max, 30 func_zoning (V1–V3 with the run's thresholds and
|
||||
tie_priority), 31 c_max — from the 13 leaf input rasters the run actually used,
|
||||
and compares the result with all 15 output rasters the run published.
|
||||
|
||||
The program is READ-ONLY: it issues only HTTP GET requests, so it can be
|
||||
pointed at any stand, production included:
|
||||
|
||||
python lp_verify.py --base-url http://localhost:18080 --run <run-id>
|
||||
python lp_verify.py --base-url https://dssgis.dwh.kz --run <run-id>
|
||||
|
||||
Data flow:
|
||||
GET /landscape/runs/{id} run params, per-node inputs and outputs
|
||||
GET /datasets/{id}.cog every input (13) and output (15) raster
|
||||
(or /datasets/{id}/download for inputs with --inputs-from download)
|
||||
|
||||
The leaf inputs are taken from the run's node `inputs` (group = slot code), the
|
||||
outputs from the node `outputs`. Expected values are computed through the whole
|
||||
chain from the leaf inputs only (E from the re-computed Z/U, C and V from the
|
||||
re-computed E, ...), so a wrong intermediate output cannot hide a later error.
|
||||
The run's own chain references (node 10 reading node 00's output, ...) are
|
||||
checked for consistency as well.
|
||||
|
||||
Large rasters (production: 22219 x 14977) are processed in windows of
|
||||
--block-rows x --block-cols pixels (default 512 x 4096, aligned with the
|
||||
512-pixel COG tiles), so memory stays bounded (~0.5 GB).
|
||||
|
||||
Pixel rules (source of truth: internal/landscape/kernels.go):
|
||||
valid(v) 1 <= v <= 5 (NoData and anything else is invalid); class
|
||||
values are used as int16(v) (truncation)
|
||||
classify(x) float32 x: < 1 -> NoData, <= 1.8 -> 1, <= 2.6 -> 2,
|
||||
<= 3.4 -> 3, <= 4.2 -> 4, <= 5.0 -> 5, else NoData
|
||||
z_integrated all Z valid: classify(0.25*Z1 + 0.25*Z2 + 0.25*Z3 + 0.25*Z4)
|
||||
(float32, summed left to right)
|
||||
u_integrated water & soil & (crop | veg): classify(num/den),
|
||||
num = 0.25*water + 0.25*soil [+ 0.25*crop] [+ 0.25*veg],
|
||||
den = 0.5 [+ 0.25] [+ 0.25] (float32)
|
||||
z_max all Z valid: max; u_max: U domain as above, max of valid
|
||||
e Z_int & U_int valid: max(Z, U)
|
||||
c_conflicts P_j & E valid: min(P_j, E)
|
||||
p_max, c_max max over valid inputs; NoData when none is valid
|
||||
func_zoning E valid & any P_k valid; candidate k: P_k valid & rule(E, P_k)
|
||||
V1: E<=2 & S>=v1_min_s_e12 | E==3 & S>=v1_min_s_e3
|
||||
V2: E<=2 | E==3 & S>=v2_min_s_e3 | E==4 & S>=v2_min_s_e4
|
||||
V3: E<=4
|
||||
no candidate -> 1; else sector with max S (ties by
|
||||
tie_priority) + 1
|
||||
|
||||
Exit code: 0 when every output matches pixel for pixel, 1 on any mismatch,
|
||||
2 on a usage / data error.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import rasterio
|
||||
from rasterio.windows import Window
|
||||
except ImportError: # pragma: no cover
|
||||
sys.exit("lp_verify.py needs numpy and rasterio: pip install numpy rasterio")
|
||||
|
||||
NODATA = np.int16(-9999)
|
||||
SLOTS = [
|
||||
"lp-z-biodiversity", "lp-z-lcr", "lp-z-soil-protection", "lp-z-wr",
|
||||
"lp-u-surf-water", "lp-u-soil", "lp-u-crop", "lp-u-veg",
|
||||
"lp-p-irrigation", "lp-p-snow", "lp-p-crop", "lp-p-pasture", "lp-p-tourism",
|
||||
]
|
||||
Z = SLOTS[0:4]
|
||||
U = SLOTS[4:8] # water, soil, crop, veg
|
||||
P = SLOTS[8:13] # sector order 1..5
|
||||
C = ["lp-c-irrigation", "lp-c-snow", "lp-c-crop", "lp-c-pasture", "lp-c-tourism"]
|
||||
OUTPUTS = [
|
||||
"lp-z-integrated", "lp-u-integrated", "lp-z-max", "lp-u-max", "lp-e",
|
||||
*C, "lp-p-max", "lp-v1-conservative", "lp-v2-balanced", "lp-v3-development", "lp-c-max",
|
||||
]
|
||||
OUTPUT_NODE = {
|
||||
"lp-z-integrated": "00", "lp-u-integrated": "01", "lp-z-max": "02", "lp-u-max": "03", "lp-e": "10",
|
||||
**{c: "20" for c in C}, "lp-p-max": "21",
|
||||
"lp-v1-conservative": "30", "lp-v2-balanced": "30", "lp-v3-development": "30", "lp-c-max": "31",
|
||||
}
|
||||
DEFAULT_PARAMS = {"tie_priority": [1, 2, 3, 4, 5], "v1_min_s_e12": 3, "v1_min_s_e3": 4,
|
||||
"v2_min_s_e3": 3, "v2_min_s_e4": 5}
|
||||
|
||||
F32 = np.float32
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- pixel rules
|
||||
def valid(v):
|
||||
return (v >= 1) & (v <= 5)
|
||||
|
||||
|
||||
def cls(v, ok):
|
||||
"""int16(v) for valid pixels (Go truncation), 0 elsewhere."""
|
||||
return np.where(ok, np.trunc(np.where(ok, v, 0)), 0).astype(np.int16)
|
||||
|
||||
|
||||
def classify(x):
|
||||
x = x.astype(np.float32, copy=False)
|
||||
return np.select(
|
||||
[x < F32(1.0), x <= F32(1.8), x <= F32(2.6), x <= F32(3.4), x <= F32(4.2), x <= F32(5.0)],
|
||||
[NODATA, 1, 2, 3, 4, 5], default=NODATA).astype(np.int16)
|
||||
|
||||
|
||||
def quarter(v):
|
||||
# float32(float32(0.25) * float32(v)), as in the Go kernels
|
||||
return F32(0.25) * v.astype(np.float32)
|
||||
|
||||
|
||||
def z_integrated(z):
|
||||
ok = valid(z[0]) & valid(z[1]) & valid(z[2]) & valid(z[3])
|
||||
score = ((quarter(z[0]) + quarter(z[1])) + quarter(z[2])) + quarter(z[3])
|
||||
return np.where(ok, classify(score), NODATA).astype(np.int16)
|
||||
|
||||
|
||||
def u_domain(u):
|
||||
water, soil, crop, veg = u
|
||||
crop_ok, veg_ok = valid(crop), valid(veg)
|
||||
return valid(water) & valid(soil) & (crop_ok | veg_ok), crop_ok, veg_ok
|
||||
|
||||
|
||||
def u_integrated(u):
|
||||
water, soil, crop, veg = u
|
||||
ok, crop_ok, veg_ok = u_domain(u)
|
||||
num = quarter(water) + quarter(soil)
|
||||
den = np.full(num.shape, F32(0.5), dtype=np.float32)
|
||||
num = np.where(crop_ok, num + quarter(crop), num).astype(np.float32)
|
||||
den = np.where(crop_ok, den + F32(0.25), den).astype(np.float32)
|
||||
num = np.where(veg_ok, num + quarter(veg), num).astype(np.float32)
|
||||
den = np.where(veg_ok, den + F32(0.25), den).astype(np.float32)
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
score = (num / den).astype(np.float32)
|
||||
return np.where(ok, classify(score), NODATA).astype(np.int16)
|
||||
|
||||
|
||||
def z_max(z):
|
||||
ok = valid(z[0]) & valid(z[1]) & valid(z[2]) & valid(z[3])
|
||||
m = np.maximum.reduce([cls(v, ok) for v in z])
|
||||
return np.where(ok, m, NODATA).astype(np.int16)
|
||||
|
||||
|
||||
def u_max(u):
|
||||
water, soil, crop, veg = u
|
||||
ok, crop_ok, veg_ok = u_domain(u)
|
||||
m = np.maximum(cls(water, ok), cls(soil, ok))
|
||||
m = np.where(crop_ok, np.maximum(m, cls(crop, crop_ok)), m)
|
||||
m = np.where(veg_ok, np.maximum(m, cls(veg, veg_ok)), m)
|
||||
return np.where(ok, m, NODATA).astype(np.int16)
|
||||
|
||||
|
||||
def e_constraint(zi, ui):
|
||||
ok = valid(zi) & valid(ui)
|
||||
return np.where(ok, np.maximum(zi, ui), NODATA).astype(np.int16)
|
||||
|
||||
|
||||
def c_conflict(p, e):
|
||||
ok = valid(p) & valid(e)
|
||||
return np.where(ok, np.minimum(cls(p, ok), e), NODATA).astype(np.int16)
|
||||
|
||||
|
||||
def max_any_valid(layers):
|
||||
oks = [valid(v) for v in layers]
|
||||
m = np.maximum.reduce([cls(v, ok) for v, ok in zip(layers, oks)])
|
||||
anyok = np.logical_or.reduce(oks)
|
||||
return np.where(anyok, m, NODATA).astype(np.int16)
|
||||
|
||||
|
||||
def func_zoning(e, p, params):
|
||||
eok = valid(e)
|
||||
ev = np.where(eok, e, 0).astype(np.int16)
|
||||
base = [valid(v) for v in p]
|
||||
s = [cls(v, ok) for v, ok in zip(p, base)]
|
||||
any_base = np.logical_or.reduce(base)
|
||||
t = params
|
||||
rules = [
|
||||
lambda e_, s_: ((e_ <= 2) & (s_ >= t["v1_min_s_e12"])) | ((e_ == 3) & (s_ >= t["v1_min_s_e3"])),
|
||||
lambda e_, s_: (e_ <= 2) | ((e_ == 3) & (s_ >= t["v2_min_s_e3"])) | ((e_ == 4) & (s_ >= t["v2_min_s_e4"])),
|
||||
lambda e_, s_: e_ <= 4,
|
||||
]
|
||||
outs = []
|
||||
for rule in rules:
|
||||
cand = [b & rule(ev, sk) for b, sk in zip(base, s)]
|
||||
any_cand = np.logical_or.reduce(cand)
|
||||
max_s = np.maximum.reduce([np.where(ck, sk, 0) for ck, sk in zip(cand, s)]).astype(np.int16)
|
||||
sector = np.zeros(ev.shape, dtype=np.int16)
|
||||
done = np.zeros(ev.shape, dtype=bool)
|
||||
for code in t["tie_priority"]:
|
||||
k = code - 1
|
||||
hit = cand[k] & (s[k] == max_s) & ~done
|
||||
sector[hit] = code
|
||||
done |= hit
|
||||
v = np.where(any_cand, sector + 1, 1).astype(np.int16)
|
||||
outs.append(np.where(eok & any_base, v, NODATA).astype(np.int16))
|
||||
return outs
|
||||
|
||||
|
||||
def expected_block(inp, params):
|
||||
"""inp: dict slot -> float64 array. Returns dict output group -> int16."""
|
||||
z = [inp[g] for g in Z]
|
||||
u = [inp[g] for g in U]
|
||||
p = [inp[g] for g in P]
|
||||
out = {
|
||||
"lp-z-integrated": z_integrated(z),
|
||||
"lp-u-integrated": u_integrated(u),
|
||||
"lp-z-max": z_max(z),
|
||||
"lp-u-max": u_max(u),
|
||||
}
|
||||
e = e_constraint(out["lp-z-integrated"], out["lp-u-integrated"])
|
||||
out["lp-e"] = e
|
||||
for cg, pk in zip(C, p):
|
||||
out[cg] = c_conflict(pk, e)
|
||||
out["lp-p-max"] = max_any_valid(p)
|
||||
v1, v2, v3 = func_zoning(e, p, params)
|
||||
out["lp-v1-conservative"], out["lp-v2-balanced"], out["lp-v3-development"] = v1, v2, v3
|
||||
out["lp-c-max"] = max_any_valid([out[c] for c in C])
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- HTTP (GET)
|
||||
def http_get(url, timeout=120, retries=3):
|
||||
req = urllib.request.Request(url, method="GET", headers={"User-Agent": "lp_verify/1.0"})
|
||||
last = None
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
return urllib.request.urlopen(req, timeout=timeout)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code < 500:
|
||||
raise
|
||||
last = e
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
|
||||
last = e
|
||||
time.sleep(2 * (attempt + 1))
|
||||
raise last
|
||||
|
||||
|
||||
def get_json(url):
|
||||
with http_get(url) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
|
||||
|
||||
def download(url, path, wait=0):
|
||||
"""GET url into path (reused when already cached). A 409 means the COG is
|
||||
still being produced (conversion is asynchronous): retry for up to `wait`
|
||||
seconds."""
|
||||
if os.path.exists(path) and os.path.getsize(path) > 0:
|
||||
return "cached"
|
||||
tmp = path + ".part"
|
||||
deadline = time.time() + wait
|
||||
while True:
|
||||
try:
|
||||
r = http_get(url, timeout=600)
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 409 or time.time() >= deadline:
|
||||
raise
|
||||
time.sleep(2)
|
||||
with r, open(tmp, "wb") as f:
|
||||
shutil.copyfileobj(r, f, length=4 << 20)
|
||||
os.replace(tmp, path)
|
||||
return "%.1f MB" % (os.path.getsize(path) / 1e6)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------- main
|
||||
def parse_points(s):
|
||||
pts = []
|
||||
for part in filter(None, (x.strip() for x in s.split(";"))):
|
||||
c, r = part.split(",")
|
||||
pts.append((int(c), int(r)))
|
||||
return pts
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:18080"))
|
||||
ap.add_argument("--run", required=True, help="landscape run id")
|
||||
ap.add_argument("--cache", help="directory for downloaded rasters (kept; existing files are reused)")
|
||||
ap.add_argument("--inputs-from", choices=["cog", "download"], default="cog",
|
||||
help="read inputs from their COG (default) or the original upload (what the node read)")
|
||||
ap.add_argument("--block-rows", type=int, default=512)
|
||||
ap.add_argument("--block-cols", type=int, default=4096)
|
||||
ap.add_argument("--wait", type=int, default=180,
|
||||
help="seconds to keep retrying a COG that is not ready yet (HTTP 409), default 180")
|
||||
ap.add_argument("--points", help='spot-check pixels "col,row;col,row;..." and print their values')
|
||||
args = ap.parse_args()
|
||||
base = args.base_url.rstrip("/")
|
||||
|
||||
run = get_json("%s/landscape/runs/%s" % (base, args.run))
|
||||
params = dict(DEFAULT_PARAMS)
|
||||
raw_params = run.get("params") or {}
|
||||
for k in DEFAULT_PARAMS:
|
||||
if raw_params.get(k):
|
||||
params[k] = raw_params[k]
|
||||
print("run %s status=%s from_stage=%s" % (run["id"], run["status"], run.get("from_stage")))
|
||||
print("params %s" % json.dumps(params))
|
||||
nodes = sorted(run.get("nodes") or [], key=lambda n: n["node_code"])
|
||||
print("nodes %s" % " ".join("%s:%s" % (n["node_code"], n["status"]) for n in nodes))
|
||||
if run["status"] != "succeeded":
|
||||
print("ERROR: run is not succeeded; nothing to verify")
|
||||
return 2
|
||||
|
||||
errors = []
|
||||
slot_ids, out_ids = {}, {}
|
||||
for n in nodes:
|
||||
for ref in n.get("inputs") or []:
|
||||
g, did = ref["group"], ref["dataset_id"]
|
||||
if g in SLOTS:
|
||||
if slot_ids.setdefault(g, did) != did:
|
||||
errors.append("slot %s resolved to two datasets (%s, %s)" % (g, slot_ids[g], did))
|
||||
for ref in n.get("outputs") or []:
|
||||
out_ids[ref["group"]] = ref["dataset_id"]
|
||||
if OUTPUT_NODE.get(ref["group"]) != n["node_code"]:
|
||||
errors.append("output %s reported by node %s" % (ref["group"], n["node_code"]))
|
||||
# chain consistency: an intermediate input must be what this run produced
|
||||
for n in nodes:
|
||||
for ref in n.get("inputs") or []:
|
||||
g = ref["group"]
|
||||
if g in OUTPUTS and out_ids.get(g) != ref["dataset_id"]:
|
||||
errors.append("node %s read %s=%s but the run's output is %s" % (n["node_code"], g, ref["dataset_id"], out_ids.get(g)))
|
||||
missing = [g for g in SLOTS if g not in slot_ids] + [g for g in OUTPUTS if g not in out_ids]
|
||||
if missing:
|
||||
errors.append("run does not reference: %s" % ", ".join(missing))
|
||||
if errors:
|
||||
for e in errors:
|
||||
print("ERROR:", e)
|
||||
return 2
|
||||
|
||||
cache = args.cache or tempfile.mkdtemp(prefix="lp_verify_")
|
||||
os.makedirs(cache, exist_ok=True)
|
||||
files = {}
|
||||
t0 = time.time()
|
||||
for g in SLOTS + OUTPUTS:
|
||||
did = slot_ids.get(g) or out_ids[g]
|
||||
if g in SLOTS and args.inputs_from == "download":
|
||||
url, path = "%s/datasets/%s/download" % (base, did), os.path.join(cache, did + ".orig.tif")
|
||||
else:
|
||||
url, path = "%s/datasets/%s.cog" % (base, did), os.path.join(cache, did + ".cog.tif")
|
||||
try:
|
||||
info = download(url, path, args.wait)
|
||||
except urllib.error.HTTPError as e:
|
||||
print("ERROR: GET %s -> HTTP %d (output not ready or missing?)" % (url, e.code))
|
||||
return 2
|
||||
files[g] = path
|
||||
print("fetch %-22s %s %s" % (g, did, info))
|
||||
print("downloaded in %.1f s to %s" % (time.time() - t0, cache))
|
||||
|
||||
with rasterio.Env(GDAL_CACHEMAX=512):
|
||||
ds = {g: rasterio.open(files[g]) for g in SLOTS + OUTPUTS}
|
||||
ref = ds[SLOTS[0]]
|
||||
for g, d in ds.items():
|
||||
if (d.width, d.height) != (ref.width, ref.height) or not d.transform.almost_equals(ref.transform) or d.crs != ref.crs:
|
||||
errors.append("%s grid %dx%d %s differs from %s %dx%d %s" % (
|
||||
g, d.width, d.height, d.crs, SLOTS[0], ref.width, ref.height, ref.crs))
|
||||
for g in OUTPUTS:
|
||||
if ds[g].dtypes[0] != "int16" or ds[g].nodata != -9999:
|
||||
errors.append("output %s: dtype %s nodata %s (want int16, -9999)" % (g, ds[g].dtypes[0], ds[g].nodata))
|
||||
if errors:
|
||||
for e in errors:
|
||||
print("ERROR:", e)
|
||||
return 2
|
||||
W, H = ref.width, ref.height
|
||||
print("grid %dx%d %s pixel %.3f" % (W, H, ref.crs, ref.transform.a))
|
||||
|
||||
if args.points:
|
||||
return spot_check(ds, parse_points(args.points), params)
|
||||
|
||||
mism = {g: 0 for g in OUTPUTS}
|
||||
validn = {g: 0 for g in OUTPUTS}
|
||||
first = {}
|
||||
t0 = time.time()
|
||||
for r0 in range(0, H, args.block_rows):
|
||||
h = min(args.block_rows, H - r0)
|
||||
for c0 in range(0, W, args.block_cols):
|
||||
w = min(args.block_cols, W - c0)
|
||||
win = Window(c0, r0, w, h)
|
||||
inp = {g: ds[g].read(1, window=win, masked=False).astype(np.float64) for g in SLOTS}
|
||||
exp = expected_block(inp, params)
|
||||
for g in OUTPUTS:
|
||||
act = ds[g].read(1, window=win, masked=False)
|
||||
bad = exp[g] != act
|
||||
nbad = int(bad.sum())
|
||||
validn[g] += int((exp[g] != NODATA).sum())
|
||||
if nbad:
|
||||
mism[g] += nbad
|
||||
if g not in first:
|
||||
rr, cc = np.argwhere(bad)[0]
|
||||
first[g] = (c0 + int(cc), r0 + int(rr), int(exp[g][rr, cc]), int(act[rr, cc]))
|
||||
print("compared in %.1f s" % (time.time() - t0))
|
||||
for d in ds.values():
|
||||
d.close()
|
||||
|
||||
print()
|
||||
print("%-20s %-4s %-38s %12s %12s %10s" % ("output", "node", "dataset_id", "pixels", "valid(exp)", "mismatch"))
|
||||
total = 0
|
||||
for g in OUTPUTS:
|
||||
total += mism[g]
|
||||
line = "%-20s %-4s %-38s %12d %12d %10d" % (g, OUTPUT_NODE[g], out_ids[g], W * H, validn[g], mism[g])
|
||||
if g in first:
|
||||
line += " first at col=%d row=%d expected=%d actual=%d" % first[g]
|
||||
print(line)
|
||||
print()
|
||||
print("TOTAL mismatching pixels: %d -> %s" % (total, "PASS" if total == 0 else "FAIL"))
|
||||
return 0 if total == 0 else 1
|
||||
|
||||
|
||||
def spot_check(ds, points, params):
|
||||
bad = 0
|
||||
shown = ["lp-z-integrated", "lp-u-integrated", "lp-e", *C, "lp-v1-conservative", "lp-v2-balanced", "lp-v3-development"]
|
||||
for col, row in points:
|
||||
win = Window(col, row, 1, 1)
|
||||
inp = {g: ds[g].read(1, window=win, masked=False).astype(np.float64) for g in SLOTS}
|
||||
exp = expected_block(inp, params)
|
||||
vals = " ".join("%s=%d" % (g.replace("lp-", ""), int(inp[g][0, 0])) for g in SLOTS)
|
||||
print("point (%d,%d) inputs: %s" % (col, row, vals))
|
||||
for g in shown:
|
||||
act = int(ds[g].read(1, window=win, masked=False)[0, 0])
|
||||
e = int(exp[g][0, 0])
|
||||
ok = act == e
|
||||
bad += 0 if ok else 1
|
||||
print(" %-20s expected=%6d actual=%6d %s" % (g, e, act, "OK" if ok else "MISMATCH"))
|
||||
print("SPOT CHECK: %d points, %d mismatches -> %s" % (len(points), bad, "PASS" if bad == 0 else "FAIL"))
|
||||
return 0 if bad == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
2
test/acceptance/requirements.txt
Normal file
2
test/acceptance/requirements.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
numpy>=1.24
|
||||
rasterio>=1.3
|
||||
131
test/acceptance/scan.sh
Executable file
131
test/acceptance/scan.sh
Executable file
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bash
|
||||
# Security re-scan (Приложение 4 «Отчет об устранении уязвимостей»):
|
||||
# 1. govulncheck, source mode — vulnerabilities reachable from the code
|
||||
# 2. govulncheck -mode=binary — the binary shipped in the image
|
||||
# 3. gosec — static analysis (HIGH severity counted)
|
||||
# 4. trivy fs — dependencies, secrets, IaC misconfig
|
||||
# 5. trivy image — the application image (built here)
|
||||
# 6. golangci-lint (if installed) — code quality, informational only
|
||||
#
|
||||
# test/acceptance/scan.sh
|
||||
#
|
||||
# Environment: SCAN_IMAGE (default gis-scan:local, built from
|
||||
# build/package/Dockerfile), SKIP_IMAGE=1 (no docker: skip 2 via image and 5;
|
||||
# the binary for 2 is then built locally with cgo + GDAL).
|
||||
# govulncheck has no severity scale: every vulnerability reachable from the
|
||||
# code (source) or present in the binary symbol table is counted as High.
|
||||
# Prints High/Critical per tool; exit code 1 if any High/Critical is found.
|
||||
# Raw reports: test/acceptance/results/scan-<timestamp>/.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$HERE/../.." && pwd)"
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
OUT="$HERE/results/scan-$STAMP"; mkdir -p "$OUT"
|
||||
SCAN_IMAGE="${SCAN_IMAGE:-gis-scan:local}"
|
||||
SKIP_IMAGE="${SKIP_IMAGE:-0}"
|
||||
GOBIN_DIR="$(go env GOPATH 2>/dev/null)/bin"
|
||||
PATH="$PATH:$GOBIN_DIR"
|
||||
cd "$ROOT"
|
||||
|
||||
ROWS=""; FAIL=0
|
||||
row() { # tool target high critical note
|
||||
ROWS="${ROWS}$(printf '%-26s | %-40s | %5s | %8s | %s' "$1" "$2" "$3" "$4" "$5")"$'\n'
|
||||
case "$3$4" in *[!0-9]*) ;; *) [ "$3" -eq 0 ] && [ "$4" -eq 0 ] || FAIL=1;; esac
|
||||
}
|
||||
have() { command -v "$1" >/dev/null 2>&1; }
|
||||
log() { echo "[$(date +%H:%M:%S)] $*"; }
|
||||
|
||||
log "commit $(git rev-parse --short HEAD 2>/dev/null || echo '?'), $(go version)"
|
||||
|
||||
# --- govulncheck (source) -------------------------------------------------
|
||||
gv_count() { # json stream -> number of distinct OSV ids with a call trace (reachable)
|
||||
jq -rs '[.[] | select(.finding != null) | .finding | select((.trace // []) | length > 0)
|
||||
| select(.trace[0].function != null) | .osv] | unique | length' "$1"
|
||||
}
|
||||
gv_ids() { jq -rs '[.[] | select(.finding != null) | .finding | select(.trace[0].function != null) | .osv] | unique | join(", ")' "$1"; }
|
||||
if have govulncheck; then
|
||||
log "govulncheck ./... (source)"
|
||||
govulncheck -format json ./... > "$OUT/govulncheck-source.json" 2> "$OUT/govulncheck-source.err" || true
|
||||
n="$(gv_count "$OUT/govulncheck-source.json")"
|
||||
mods="$(jq -rs '[.[] | select(.finding != null) | .finding.osv] | unique | length' "$OUT/govulncheck-source.json")"
|
||||
row "govulncheck (source)" "./..." "$n" 0 "reachable: ${n}$( [ "$n" -gt 0 ] && echo " ($(gv_ids "$OUT/govulncheck-source.json"))"); in dependencies but not called: $((mods - n))"
|
||||
else row "govulncheck (source)" "./..." "n/a" "n/a" "not installed (go install golang.org/x/vuln/cmd/govulncheck@latest)"; fi
|
||||
|
||||
# --- application image + binary -------------------------------------------
|
||||
BIN=""
|
||||
if [ "$SKIP_IMAGE" != 1 ] && have docker; then
|
||||
log "docker build -t $SCAN_IMAGE -f build/package/Dockerfile ."
|
||||
docker build -q -t "$SCAN_IMAGE" -f build/package/Dockerfile . > "$OUT/docker-build.txt"
|
||||
cid="$(docker create "$SCAN_IMAGE")"
|
||||
docker cp "$cid:/usr/local/bin/gis" "$OUT/gis" >/dev/null; docker rm "$cid" >/dev/null
|
||||
BIN="$OUT/gis"; BIN_SRC="image $SCAN_IMAGE:/usr/local/bin/gis"
|
||||
else
|
||||
log "go build (cgo + GDAL) for binary-mode scan"
|
||||
CGO_ENABLED=1 go build -trimpath -o "$OUT/gis" ./cmd/gis; BIN="$OUT/gis"; BIN_SRC="local build"
|
||||
fi
|
||||
|
||||
# --- govulncheck (binary) -------------------------------------------------
|
||||
if have govulncheck; then
|
||||
log "govulncheck -mode=binary ($BIN_SRC)"
|
||||
govulncheck -mode=binary -format json "$BIN" > "$OUT/govulncheck-binary.json" 2> "$OUT/govulncheck-binary.err" || true
|
||||
n="$(gv_count "$OUT/govulncheck-binary.json")"
|
||||
gov="$(go version "$BIN" 2>/dev/null | awk '{print $2}')"
|
||||
row "govulncheck (binary)" "$BIN_SRC" "$n" 0 "built with $gov; vulnerable symbols present: $n$( [ "$n" -gt 0 ] && echo " ($(gv_ids "$OUT/govulncheck-binary.json"))")"
|
||||
fi
|
||||
|
||||
# --- gosec -----------------------------------------------------------------
|
||||
if have gosec; then
|
||||
log "gosec ./..."
|
||||
gosec -quiet -fmt json -out "$OUT/gosec.json" ./... 2> "$OUT/gosec.err" || true
|
||||
hi="$(jq '[.Issues[]? | select(.severity=="HIGH")] | length' "$OUT/gosec.json")"
|
||||
me="$(jq '[.Issues[]? | select(.severity=="MEDIUM")] | length' "$OUT/gosec.json")"
|
||||
lo="$(jq '[.Issues[]? | select(.severity=="LOW")] | length' "$OUT/gosec.json")"
|
||||
ns="$(jq '.Stats.nosec // 0' "$OUT/gosec.json")"
|
||||
row "gosec" "./... ($(jq '.Stats.files' "$OUT/gosec.json") files)" "$hi" 0 "MEDIUM $me, LOW $lo; #nosec-annotated $ns"
|
||||
else row "gosec" "./..." "n/a" "n/a" "not installed"; fi
|
||||
|
||||
# --- trivy ------------------------------------------------------------------
|
||||
trivy_counts() { # json -> "high critical"
|
||||
jq -r '[.Results[]? | (.Vulnerabilities[]?, .Secrets[]?, .Misconfigurations[]?) | .Severity] as $s
|
||||
| "\([$s[] | select(.=="HIGH")] | length) \([$s[] | select(.=="CRITICAL")] | length)"' "$1"
|
||||
}
|
||||
if have trivy; then
|
||||
log "trivy fs (vuln, secret, misconfig)"
|
||||
trivy fs --quiet --scanners vuln,secret,misconfig --format json --output "$OUT/trivy-fs.json" \
|
||||
--skip-dirs test/acceptance/results --skip-dirs graft . 2> "$OUT/trivy-fs.err" || true
|
||||
read -r h c <<< "$(trivy_counts "$OUT/trivy-fs.json")"
|
||||
all="$(jq '[.Results[]? | (.Vulnerabilities[]?, .Secrets[]?, .Misconfigurations[]?)] | length' "$OUT/trivy-fs.json")"
|
||||
row "trivy fs" "repository (go.mod, Dockerfile, compose)" "$h" "$c" "all severities: $all"
|
||||
if [ -n "$BIN" ] && [ "$SKIP_IMAGE" != 1 ] && have docker; then
|
||||
log "trivy image $SCAN_IMAGE"
|
||||
trivy image --quiet --format json --output "$OUT/trivy-image.json" "$SCAN_IMAGE" 2> "$OUT/trivy-image.err" || true
|
||||
read -r h c <<< "$(trivy_counts "$OUT/trivy-image.json")"
|
||||
all="$(jq '[.Results[]? | .Vulnerabilities[]?] | length' "$OUT/trivy-image.json")"
|
||||
os="$(jq -r '.Metadata.OS | "\(.Family) \(.Name)"' "$OUT/trivy-image.json")"
|
||||
row "trivy image" "$SCAN_IMAGE ($os)" "$h" "$c" "all severities: $all"
|
||||
fi
|
||||
else row "trivy" "-" "n/a" "n/a" "not installed"; fi
|
||||
|
||||
# --- golangci-lint (informational) ------------------------------------------
|
||||
if have golangci-lint; then
|
||||
log "golangci-lint run ./..."
|
||||
note=""
|
||||
golangci-lint run --output.json.path "$OUT/golangci-lint.json" ./... > "$OUT/golangci-lint.txt" 2>&1 || true
|
||||
if grep -q "unsupported version of the configuration" "$OUT/golangci-lint.txt"; then
|
||||
# .golangci.yml is in the v1 format; a v2 binary refuses it — use the default linters
|
||||
note=" (.golangci.yml is v1 format, $(golangci-lint version 2>/dev/null | awk '{print $4}') ran with --no-config)"
|
||||
golangci-lint run --no-config --output.json.path "$OUT/golangci-lint.json" ./... > "$OUT/golangci-lint.txt" 2>&1 || true
|
||||
fi
|
||||
n="$(jq '[.Issues[]?] | length' "$OUT/golangci-lint.json" 2>/dev/null || echo '?')"
|
||||
row "golangci-lint" "./..." "-" "-" "issues: $n$note; no severity scale, informational"
|
||||
fi
|
||||
|
||||
echo
|
||||
printf '%-26s | %-40s | %5s | %8s | %s\n' "Tool" "Target" "High" "Critical" "Notes"
|
||||
printf '%s\n' "---------------------------+------------------------------------------+-------+----------+------------------------------"
|
||||
printf '%s' "$ROWS"
|
||||
echo
|
||||
if [ "$FAIL" = 0 ]; then echo "RESULT: 0 High / 0 Critical -> PASS"; else echo "RESULT: High/Critical findings present -> FAIL"; fi
|
||||
echo "raw reports: $OUT"
|
||||
exit "$FAIL"
|
||||
101
test/acceptance/seed.sh
Executable file
101
test/acceptance/seed.sh
Executable file
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env bash
|
||||
# Seeds the acceptance stand with datasets for the load test (Приложение 3,
|
||||
# п. 3.3), so the read endpoints work on a realistic catalogue.
|
||||
#
|
||||
# test/acceptance/seed.sh create (idempotent) category "acc-load":
|
||||
# SEED_VECTOR (25) vector GeoJSON,
|
||||
# SEED_KATO (25) vector_with_kato GeoJSON with
|
||||
# KATO mapping (14 districts x 3 years each),
|
||||
# SEED_RASTER (50) GeoTIFF rasters; the
|
||||
# vector_with_kato datasets form the group
|
||||
# "acc-load-population"
|
||||
# test/acceptance/seed.sh clean delete everything seed.sh created
|
||||
#
|
||||
# Writes the ids load.sh needs to test/acceptance/results/seed.env.
|
||||
# WRITES to the stand: never point it at production.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
BASE_URL="${BASE_URL:-http://localhost:18080}"
|
||||
BASE_URL="${BASE_URL%/}"
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
SEED_VECTOR="${SEED_VECTOR:-25}"
|
||||
SEED_KATO="${SEED_KATO:-25}"
|
||||
SEED_RASTER="${SEED_RASTER:-50}"
|
||||
CAT_CODE=acc-load
|
||||
GRP_CODE=acc-load-population
|
||||
ENV_OUT="$HERE/results/seed.env"
|
||||
mkdir -p "$HERE/results"
|
||||
case "$BASE_URL" in
|
||||
*dssgis.dwh.kz*) echo "refusing to write to $BASE_URL (production)" >&2; exit 2;;
|
||||
esac
|
||||
J=(-H 'Content-Type: application/json')
|
||||
|
||||
cat_id() { curl -fsS "$BASE_URL/categories" | jq -r --arg c "$CAT_CODE" '.[]|select(.code==$c)|.id' | head -n1; }
|
||||
grp_id() { curl -fsS "$BASE_URL/dataset-groups?code=$GRP_CODE" | jq -r '.data[0].id // empty'; }
|
||||
ids_in_category() { # all dataset ids of a category (paginated)
|
||||
local cat="$1" page=1 out
|
||||
while :; do
|
||||
out="$(curl -fsS "$BASE_URL/datasets?category_id=$cat&page_size=100&page=$page")"
|
||||
jq -r '.data[].id' <<< "$out"
|
||||
[ "$page" -lt "$(jq -r .total_pages <<< "$out")" ] || break
|
||||
page=$((page + 1))
|
||||
done
|
||||
}
|
||||
|
||||
if [ "${1:-}" = clean ]; then
|
||||
CAT="$(cat_id || true)"; GRP="$(grp_id || true)"
|
||||
if [ -n "$CAT" ]; then
|
||||
n=0; for id in $(ids_in_category "$CAT"); do curl -sS -o /dev/null -X DELETE "$BASE_URL/datasets/$id"; n=$((n + 1)); done
|
||||
echo "deleted $n datasets"
|
||||
fi
|
||||
[ -n "$GRP" ] && curl -sS -o /dev/null -w "group delete %{http_code}\n" -X DELETE "$BASE_URL/dataset-groups/$GRP"
|
||||
[ -n "$CAT" ] && curl -sS -o /dev/null -w "category delete %{http_code}\n" -X DELETE "$BASE_URL/categories/$CAT"
|
||||
rm -f "$ENV_OUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CAT="$(cat_id || true)"
|
||||
if [ -z "$CAT" ]; then
|
||||
CAT="$(curl -fsS -X POST "$BASE_URL/categories" "${J[@]}" -d "{\"code\":\"$CAT_CODE\",\"name\":\"Нагрузочное тестирование\",\"description\":\"Данные для test/acceptance/load.sh\"}" | jq -r .id)"
|
||||
fi
|
||||
GRP="$(grp_id || true)"
|
||||
if [ -z "$GRP" ]; then
|
||||
GRP="$(curl -fsS -X POST "$BASE_URL/dataset-groups" "${J[@]}" -d "{\"code\":\"$GRP_CODE\",\"name\":\"Население СКО (нагрузочный тест)\"}" | jq -r .id)"
|
||||
fi
|
||||
|
||||
want=$((SEED_VECTOR + SEED_KATO + SEED_RASTER))
|
||||
have="$(curl -fsS "$BASE_URL/datasets?category_id=$CAT&status=ready&page_size=1" | jq -r .total)"
|
||||
if [ "$have" -lt "$want" ]; then
|
||||
echo "seeding $BASE_URL: $SEED_VECTOR vector, $SEED_KATO vector_with_kato, $SEED_RASTER raster (category $CAT_CODE)"
|
||||
TMP="$(mktemp -d "${TMPDIR:-/tmp}/gis-seed.XXXXXX")"; trap 'rm -rf "$TMP"' EXIT
|
||||
"$PYTHON" "$HERE/lib/helpers.py" ndvi "$TMP/ndvi_seed.tif"
|
||||
upload() { curl -fsS -X POST "$BASE_URL/datasets" "$@" | jq -r .id; }
|
||||
for i in $(seq 1 "$SEED_VECTOR"); do
|
||||
upload -F "file=@$HERE/data/plots_nko.geojson" -F file_type=vector -F "category_id=$CAT" -F "name=Участки $i" >/dev/null
|
||||
done
|
||||
kato_ids=""
|
||||
for i in $(seq 1 "$SEED_KATO"); do
|
||||
y=$((2000 + i))
|
||||
kato_ids="$kato_ids $(upload -F "file=@$HERE/data/population_kato.geojson" -F file_type=vector_with_kato -F "category_id=$CAT" \
|
||||
-F "name=Население $i" -F "dg_id=$GRP" -F "start_date=$y-01-01" -F "end_date=$y-12-31")"
|
||||
done
|
||||
for i in $(seq 1 "$SEED_RASTER"); do
|
||||
upload -F "file=@$TMP/ndvi_seed.tif" -F file_type=raster -F "category_id=$CAT" -F "name=NDVI $i" >/dev/null
|
||||
done
|
||||
MAP='{"kato_column":"kato","year_columns":[{"column":"Y2022","start_date":"2022-01-01","end_date":"2022-12-31"},{"column":"Y2023","start_date":"2023-01-01","end_date":"2023-12-31"},{"column":"Y2024","start_date":"2024-01-01","end_date":"2024-12-31"}]}'
|
||||
for id in $kato_ids; do
|
||||
for _ in $(seq 1 120); do
|
||||
[ "$(curl -fsS "$BASE_URL/datasets/$id" | jq -r .status)" = awaiting_mapping ] && break; sleep 1
|
||||
done
|
||||
curl -fsS -o /dev/null -X POST "$BASE_URL/datasets/$id/mapping" "${J[@]}" -d "$MAP"
|
||||
done
|
||||
for _ in $(seq 1 300); do
|
||||
have="$(curl -fsS "$BASE_URL/datasets?category_id=$CAT&status=ready&page_size=1" | jq -r .total)"
|
||||
[ "$have" -ge "$want" ] && break; sleep 1
|
||||
done
|
||||
fi
|
||||
VWK="$(curl -fsS "$BASE_URL/datasets?category_id=$CAT&file_type=vector_with_kato&status=ready&page_size=1" | jq -r '.data[0].id')"
|
||||
printf 'CAT_ID=%s\nGROUP_ID=%s\nVWK_ID=%s\n' "$CAT" "$GRP" "$VWK" > "$ENV_OUT"
|
||||
echo "seed: category $CAT_CODE has $have ready datasets (wanted $want); ids in $ENV_OUT"
|
||||
cat "$ENV_OUT"
|
||||
54
test/acceptance/stand.sh
Executable file
54
test/acceptance/stand.sh
Executable file
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# Brings the isolated acceptance stand (docker compose project gis-acceptance)
|
||||
# up or down.
|
||||
#
|
||||
# test/acceptance/stand.sh up build the image from this checkout, start
|
||||
# all services, apply migrations, import the
|
||||
# 14 districts, wait for /readyz
|
||||
# test/acceptance/stand.sh down stop and remove the stand with its volumes
|
||||
# test/acceptance/stand.sh ps|logs pass-through to docker compose
|
||||
#
|
||||
# Environment: MINIO_IMAGE, POSTGIS_PLATFORM, APP_IMAGE, HTTP_MAX_UPLOAD_BYTES,
|
||||
# LP_MAX_WORKERS, LP_MAX_NODES (see compose.acceptance.yml).
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$HERE/../.." && pwd)"
|
||||
PROJECT=gis-acceptance
|
||||
compose() {
|
||||
docker compose -p "$PROJECT" -f "$ROOT/deployments/docker-compose.yml" -f "$HERE/compose.acceptance.yml" "$@"
|
||||
}
|
||||
|
||||
cmd="${1:-up}"
|
||||
shift || true
|
||||
case "$cmd" in
|
||||
up)
|
||||
echo "== build application image from $(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null || echo 'this checkout')"
|
||||
compose build app worker
|
||||
echo "== start infrastructure ($PROJECT)"
|
||||
compose up -d --wait postgres minio rabbitmq
|
||||
echo "== migrate up"
|
||||
compose run --rm --no-deps app migrate up
|
||||
echo "== import districts"
|
||||
compose run --rm --no-deps app import-districts /stubs/districts.geojson
|
||||
compose up -d app worker
|
||||
echo "== wait for http://localhost:18080/readyz"
|
||||
for _ in $(seq 1 60); do
|
||||
if curl -fsS http://localhost:18080/readyz >/dev/null 2>&1; then
|
||||
curl -sS http://localhost:18080/readyz; echo
|
||||
compose ps --format 'table {{.Service}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "stand did not become ready" >&2
|
||||
compose logs --tail 50 app worker >&2
|
||||
exit 1
|
||||
;;
|
||||
down)
|
||||
compose down -v --remove-orphans
|
||||
;;
|
||||
*)
|
||||
compose "$cmd" "$@"
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user