Compare commits
25 Commits
6113202704
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 2544d4e146 | |||
| dae9294b4d | |||
| d06d67436b | |||
| 4051506b95 | |||
| bcda76003c | |||
| 6aec00ee29 | |||
| ad3c9fbdcd | |||
| d74277552f | |||
| 5fba77b59c | |||
| ff6d5fc412 | |||
| bc524f3d98 | |||
| 32eba016b3 | |||
| 831ddd1154 | |||
| 922812b765 | |||
| 37d4f3519b | |||
| 5ca6d9e662 | |||
| 91c2cab66e | |||
| bea7e116a0 | |||
| c930f5928b | |||
| 1e0844ca82 | |||
| 00a0cd1b8e | |||
| 47b2e8653d | |||
| 017f869380 | |||
| d5c291fb9b | |||
| ec8a014e82 |
@@ -9,4 +9,8 @@ S3_SECRET_KEY: minioadmin
|
|||||||
S3_BUCKET: geofiles
|
S3_BUCKET: geofiles
|
||||||
S3_USE_SSL: "false"
|
S3_USE_SSL: "false"
|
||||||
|
|
||||||
DB_PORT=5423
|
DB_PORT=5423
|
||||||
|
|
||||||
|
# Landscape Program parallelism (bounds worker peak memory)
|
||||||
|
LP_MAX_WORKERS=8
|
||||||
|
LP_MAX_NODES=2
|
||||||
|
|||||||
230
README.md
230
README.md
@@ -19,6 +19,8 @@ internal/
|
|||||||
transport/http/ chi router, middleware, handlers
|
transport/http/ chi router, middleware, handlers
|
||||||
storage/s3/ MinIO/S3 object storage
|
storage/s3/ MinIO/S3 object storage
|
||||||
messaging/rabbitmq/ connection, publisher, consumer
|
messaging/rabbitmq/ connection, publisher, consumer
|
||||||
|
landscape/ Landscape Program graph, kernels, window executor
|
||||||
|
raster/ GDAL raster I/O (COG conversion + godal engine)
|
||||||
platform/logger/ slog setup
|
platform/logger/ slog setup
|
||||||
pkg/httputil/ generic JSON/validation HTTP helpers
|
pkg/httputil/ generic JSON/validation HTTP helpers
|
||||||
migrations/ embedded goose SQL migrations
|
migrations/ embedded goose SQL migrations
|
||||||
@@ -32,6 +34,9 @@ api/openapi.yaml OpenAPI 3.1.1 spec (embedded + served at /openapi.yaml)
|
|||||||
|
|
||||||
- **Category** — hierarchical (self-referencing `parent_id`). Full CRUD; cycle-safe
|
- **Category** — hierarchical (self-referencing `parent_id`). Full CRUD; cycle-safe
|
||||||
on update.
|
on update.
|
||||||
|
- **DatasetGroup** — a named group of "одноимённые" datasets (see *Dataset
|
||||||
|
groups* below). Full CRUD; deleting a group still referenced by datasets is
|
||||||
|
rejected with 409.
|
||||||
- **Dataset** — a geo file uploaded to S3/MinIO (`file_type`: `vector_with_kato |
|
- **Dataset** — a geo file uploaded to S3/MinIO (`file_type`: `vector_with_kato |
|
||||||
vector | raster`), belonging to one Category. Carries `code`/`name`/`description`/
|
vector | raster`), belonging to one Category. Carries `code`/`name`/`description`/
|
||||||
`unit` metadata, a user-defined `meta` (JSONB) blob, an `automated` flag, a
|
`unit` metadata, a user-defined `meta` (JSONB) blob, an `automated` flag, a
|
||||||
@@ -72,10 +77,10 @@ the KATO column and the year columns:
|
|||||||
`parse_error`).
|
`parse_error`).
|
||||||
3. The client polls `GET /datasets/{id}` until `awaiting_mapping`, then submits
|
3. The client polls `GET /datasets/{id}` until `awaiting_mapping`, then submits
|
||||||
`POST /datasets/{id}/mapping` with the chosen `kato_column` and a
|
`POST /datasets/{id}/mapping` with the chosen `kato_column` and a
|
||||||
`year_columns` map (each `{column, date}`). Validated against the detected
|
`year_columns` map (each `{column, start_date}`, optionally `end_date` for
|
||||||
columns; `status` → `extracting`.
|
a period). Validated against the detected columns; `status` → `extracting`.
|
||||||
4. A second worker job **unpivots** the attribute table into long-format
|
4. A second worker job **unpivots** the attribute table into long-format
|
||||||
`dataset_observations` — one row per `(kato_code, date)` with a numeric
|
`dataset_observations` — one row per `(kato_code, start_date)` with a numeric
|
||||||
`value` (or `value_text` for non-numeric cells); `status` → `ready`. Read
|
`value` (or `value_text` for non-numeric cells); `status` → `ready`. Read
|
||||||
them via `GET /datasets/{id}/observations` (paginated, optional
|
them via `GET /datasets/{id}/observations` (paginated, optional
|
||||||
`?kato_code=`).
|
`?kato_code=`).
|
||||||
@@ -84,12 +89,191 @@ the KATO column and the year columns:
|
|||||||
curl -X POST https://dssgis.dwh.kz/datasets/<id>/mapping -H 'Content-Type: application/json' -d '{
|
curl -X POST https://dssgis.dwh.kz/datasets/<id>/mapping -H 'Content-Type: application/json' -d '{
|
||||||
"kato_column": "като",
|
"kato_column": "като",
|
||||||
"year_columns": [
|
"year_columns": [
|
||||||
{"column": "F_2023", "date": "2023-01-01"},
|
{"column": "F_2023", "start_date": "2023-01-01"},
|
||||||
{"column": "D_2025", "date": "2025-01-01"}
|
{"column": "D_2025", "start_date": "2025-01-01"}
|
||||||
]
|
]
|
||||||
}'
|
}'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Dataset periods
|
||||||
|
|
||||||
|
A dataset carries an optional period — `start_date` and `end_date`
|
||||||
|
(`YYYY-MM-DD`, both nullable, `end_date` inclusive and never before
|
||||||
|
`start_date`) — plus an optional `dg_id` linking it to a *dataset group*: the
|
||||||
|
"одноимённые" datasets that describe the same thing for different periods. All
|
||||||
|
three are accepted on `POST /datasets` (form fields), `PUT /datasets/{id}` and
|
||||||
|
`PATCH /datasets/{id}` (an explicit `null`, or omission on PUT, clears them).
|
||||||
|
|
||||||
|
`GET /datasets` filters on them with `?dg_id=`, and with the inclusive bounds
|
||||||
|
`?start_date_from=`, `?start_date_to=`, `?end_date_from=` and `?end_date_to=`
|
||||||
|
(each `YYYY-MM-DD`).
|
||||||
|
|
||||||
|
### Dataset groups
|
||||||
|
|
||||||
|
A **dataset group** (`/dataset-groups`) collects the datasets that describe the
|
||||||
|
same indicator for different periods — the "одноимённые" datasets. A group has a
|
||||||
|
slug `code` (unique), a `name` and a `description`, and datasets join it through
|
||||||
|
their `dg_id`.
|
||||||
|
|
||||||
|
**Freshness.** `GET /dataset-groups/{id}/latest` returns the group's freshest
|
||||||
|
dataset: the greatest `end_date` wins, datasets without an `end_date` come last,
|
||||||
|
and ties are broken by the newest `created_at`. Narrow the candidates with
|
||||||
|
`?status=` and `?file_type=` (for example `?status=ready&file_type=raster` to
|
||||||
|
get the latest published raster). A group with no matching dataset yields 404.
|
||||||
|
|
||||||
|
**Shared SLD.** A group can carry one SLD (Styled Layer Descriptor) style file
|
||||||
|
(`POST`/`GET /dataset-groups/{id}/sld`, multipart `file`, `.sld` or `.xml`),
|
||||||
|
which acts as the fallback style for every dataset in the group:
|
||||||
|
`GET /datasets/{id}/sld` returns the dataset's own SLD when it has one, and
|
||||||
|
otherwise falls back to its group's — 404 only when neither exists. Deleting a
|
||||||
|
group also removes its stored SLD object.
|
||||||
|
|
||||||
|
## Landscape Program (ЛП)
|
||||||
|
|
||||||
|
The **Landscape Program** is a DAG of per-pixel raster computations that turns
|
||||||
|
ecological value (Z), vulnerability (U) and sector suitability (P) class rasters
|
||||||
|
into an ecological constraint (E), per-sector conflicts (C) and three functional
|
||||||
|
zoning scenarios (V1–V3). Every input and output of every node is a *dataset
|
||||||
|
group*; the leaf inputs are the **slots** users upload to, and every result is an
|
||||||
|
ordinary `raster` dataset (COG + SLD) served by the existing dataset endpoints.
|
||||||
|
|
||||||
|
The graph lives in Go (`internal/landscape/graph.go`); the groups and the
|
||||||
|
`landscape` category are seeded by migration `00011_landscape.sql`. A node code
|
||||||
|
`XY` reads as `X` = stage, `Y` = function within the stage: a stage starts only
|
||||||
|
after every node of all lower stages finished, and nodes of one stage run
|
||||||
|
concurrently.
|
||||||
|
|
||||||
|
| Node | Name | Inputs (group codes) | Outputs (group codes) |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 00 | z_integrated | `lp-z-biodiversity`, `lp-z-lcr`, `lp-z-soil-protection`, `lp-z-wr` | `lp-z-integrated` |
|
||||||
|
| 01 | u_integrated | `lp-u-surf-water`, `lp-u-soil`, `lp-u-crop`, `lp-u-veg` | `lp-u-integrated` |
|
||||||
|
| 02 | z_max | Z1..Z4 as above | `lp-z-max` |
|
||||||
|
| 03 | u_max | U as above | `lp-u-max` |
|
||||||
|
| 10 | e_ecological_constraint | `lp-z-integrated`, `lp-u-integrated` | `lp-e` |
|
||||||
|
| 20 | c_conflicts | `lp-p-irrigation`, `lp-p-snow`, `lp-p-crop`, `lp-p-pasture`, `lp-p-tourism`, `lp-e` | `lp-c-irrigation`, `lp-c-snow`, `lp-c-crop`, `lp-c-pasture`, `lp-c-tourism` |
|
||||||
|
| 21 | p_max | five P groups | `lp-p-max` |
|
||||||
|
| 30 | func_zoning | `lp-e`, five P groups (original suitability) | `lp-v1-conservative`, `lp-v2-balanced`, `lp-v3-development` |
|
||||||
|
| 31 | c_max | five C groups | `lp-c-max` |
|
||||||
|
|
||||||
|
Sector order everywhere: 1 irrigation, 2 snow, 3 crop, 4 pasture, 5 tourism.
|
||||||
|
|
||||||
|
### Pixel rules
|
||||||
|
|
||||||
|
A pixel value is **valid** iff `1 <= v <= 5`; anything else (nodata included) is
|
||||||
|
invalid. Output rasters are `int16` with nodata `-9999`. `classify(x)` maps a
|
||||||
|
float32 score to a class: `<= 1.8 → 1`, `<= 2.6 → 2`, `<= 3.4 → 3`, `<= 4.2 → 4`,
|
||||||
|
`<= 5.0 → 5` (and requires `x >= 1.0`).
|
||||||
|
|
||||||
|
| Node | Valid domain (pixel) | Value |
|
||||||
|
|---|---|---|
|
||||||
|
| z_integrated | all four Z valid | `classify(0.25*Z1 + 0.25*Z2 + 0.25*Z3 + 0.25*Z4)` |
|
||||||
|
| u_integrated | water ∧ soil ∧ (crop ∨ veg) | `classify(num/den)` with crop/veg contributing `0.25` each only when valid |
|
||||||
|
| z_max / u_max | as above | `max` over the valid inputs |
|
||||||
|
| e_ecological_constraint | Z ∧ U valid | `max(Z, U)` |
|
||||||
|
| c_conflicts | `P_j` ∧ E valid | `min(P_j, E)` per sector |
|
||||||
|
| p_max / c_max | at least one input valid | `max` over the valid ones |
|
||||||
|
| func_zoning | E valid ∧ at least one `P_k` valid | winning sector + 1, else 1 |
|
||||||
|
|
||||||
|
Functional zoning takes `S_k = P_k`, the original suitability rasters (the conflict
|
||||||
|
layers `C_k` are diagnostics and are not zoning inputs). A sector is a candidate when `S_k` and `E`
|
||||||
|
are valid and the scenario rule holds — V1: `E <= 2 && S >= v1_min_s_e12` or
|
||||||
|
`E == 3 && S >= v1_min_s_e3`; V2: `E <= 2`, or `E == 3 && S >= v2_min_s_e3`, or
|
||||||
|
`E == 4 && S >= v2_min_s_e4`; V3: `E <= 4`. With no candidate the pixel is class
|
||||||
|
1 (regulated); otherwise the candidate with the greatest `S` wins, ties broken by
|
||||||
|
`tie_priority`, and the class is `sector + 1` (2 irrigation, 3 snow, 4 crop,
|
||||||
|
5 pasture, 6 tourism).
|
||||||
|
|
||||||
|
Inputs are **not** resampled or reprojected: a node whose inputs do not share one
|
||||||
|
grid (size, geotransform, CRS) fails with a clear message, and so does the run.
|
||||||
|
|
||||||
|
### Uploading inputs, running, reading results
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# 1. fill a slot (repeat per input group; see GET /landscape/slots for the codes)
|
||||||
|
curl -X POST https://dssgis.dwh.kz/landscape/slots/lp-z-biodiversity \
|
||||||
|
-F file=@z_biodiversity.tif -F name="Z Biodiversity 2025" \
|
||||||
|
-F start_date=2025-01-01 -F end_date=2025-12-31
|
||||||
|
|
||||||
|
# 2. queue a run (202); the worker executes the graph asynchronously
|
||||||
|
curl -X POST https://dssgis.dwh.kz/landscape/runs \
|
||||||
|
-H 'Content-Type: application/json' -d '{"from_stage": 0}'
|
||||||
|
|
||||||
|
# 3. follow it, then read the layers
|
||||||
|
curl https://dssgis.dwh.kz/landscape/runs/<run-id>
|
||||||
|
curl https://dssgis.dwh.kz/landscape/results
|
||||||
|
```
|
||||||
|
|
||||||
|
`GET /landscape/slots` lists every leaf slot with the dataset currently filling
|
||||||
|
it (any status), so the UI can show what is still missing. The upload is the
|
||||||
|
regular raster path: `.tif`/`.tiff` with a TIFF magic-byte check, stored in the
|
||||||
|
slot's group under the `landscape` category, COG conversion enqueued.
|
||||||
|
|
||||||
|
**Input resolution.** For each input group a node takes, in order: the run's
|
||||||
|
`overrides[group]`, else whatever this run already produced for that group, else
|
||||||
|
the freshest `ready` raster of the group (greatest `end_date`, NULLs last, newest
|
||||||
|
`created_at` first). A missing input fails the node and the run.
|
||||||
|
|
||||||
|
**Incremental recompute.** Each node is fingerprinted over its code, its resolved
|
||||||
|
inputs and the params that actually affect it. Without `from_stage`, a node whose
|
||||||
|
fingerprint matches an earlier `succeeded` node — with all of its output datasets
|
||||||
|
still present and `ready` — is marked `skipped` and its outputs reused. With
|
||||||
|
`from_stage: N`, every node of stage `N` and above is recomputed regardless;
|
||||||
|
`from_stage: 0` recomputes everything. `overrides` pins a specific dataset
|
||||||
|
(existing, `raster`, `ready`) for a group instead of resolving the freshest one.
|
||||||
|
Run params: `window_size` (256..8192, default 2048), `tie_priority` (a
|
||||||
|
permutation of 1..5, default `[1,2,3,4,5]`), `v1_min_s_e12`/`v1_min_s_e3`
|
||||||
|
(defaults 3/4) and `v2_min_s_e3`/`v2_min_s_e4` (defaults 3/5).
|
||||||
|
|
||||||
|
**Results.** Every output is a normal dataset in its group, named
|
||||||
|
`<Output name> <run date>`, `automated = true`, with
|
||||||
|
`meta = {lp_run_id, lp_node, inputs}` and the min/max period of its inputs.
|
||||||
|
`GET /landscape/results` returns one entry per output group with its freshest
|
||||||
|
`ready` dataset, a `cog_url` (`/datasets/{id}.cog`) and, when the group carries
|
||||||
|
one, an `sld_url` (`/dataset-groups/{id}/sld`). A node failure fails its stage;
|
||||||
|
later stages are not started and the run ends `failed` with the message. Jobs are
|
||||||
|
nacked without requeue, so there is no automatic retry — start a new run.
|
||||||
|
|
||||||
|
### Memory and parallelism
|
||||||
|
|
||||||
|
A run has two nested levels of parallelism: nodes of a stage run concurrently,
|
||||||
|
and each node processes its windows with its own worker pool. Every worker holds
|
||||||
|
one window buffer per input **and** per output, so the worker's peak resident set
|
||||||
|
is roughly
|
||||||
|
|
||||||
|
```
|
||||||
|
nodes × workers-per-node × window_size² × (inputs + outputs) × 8 bytes
|
||||||
|
```
|
||||||
|
|
||||||
|
(the widest node, `20 c_conflicts`, has 6 inputs and 5 outputs). Two environment
|
||||||
|
variables bound that product, independently of how many cores the host has:
|
||||||
|
|
||||||
|
| Variable | Default | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `LP_MAX_WORKERS` | `8` | Total window-worker budget shared by all nodes of a run. |
|
||||||
|
| `LP_MAX_NODES` | `2` | How many graph nodes of one stage run at the same time. |
|
||||||
|
|
||||||
|
Each node gets `max(1, LP_MAX_WORKERS / LP_MAX_NODES)` workers, further capped by
|
||||||
|
`GOMAXPROCS` and by the number of windows, so the total stays at
|
||||||
|
`LP_MAX_WORKERS`. At the defaults and `window_size: 2048` that is about
|
||||||
|
8 × 2048² × 11 × 8 B ≈ 2.9 GiB worst case — lower `LP_MAX_WORKERS` (or
|
||||||
|
`window_size`) on a small host, raise it on a large one.
|
||||||
|
|
||||||
|
### Build requirements
|
||||||
|
|
||||||
|
The raster math uses [godal](https://github.com/airbusgeo/godal), a **cgo**
|
||||||
|
binding to GDAL, so the binary must be built with `CGO_ENABLED=1` against GDAL
|
||||||
|
headers and libraries:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
brew install gdal # macOS; Linux: libgdal-dev / gdal-dev
|
||||||
|
go build ./... # cgo is on by default for a native build
|
||||||
|
```
|
||||||
|
|
||||||
|
`build/package/Dockerfile` already handles the image: the builder stage adds
|
||||||
|
`gdal-dev gcc g++ musl-dev pkgconfig` and builds with `CGO_ENABLED=1`, and the
|
||||||
|
runtime stage installs `gdal` (the shared library godal needs) plus `gdal-tools`
|
||||||
|
(`gdal_translate`/`gdalinfo` for the COG conversion).
|
||||||
|
|
||||||
## Getting started
|
## Getting started
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -117,16 +301,31 @@ server runs it is served at `/openapi.yaml`, with an interactive **Redoc** UI at
|
|||||||
| GET | `/categories/{id}` | get |
|
| GET | `/categories/{id}` | get |
|
||||||
| PUT | `/categories/{id}` | update |
|
| PUT | `/categories/{id}` | update |
|
||||||
| DELETE | `/categories/{id}` | delete |
|
| DELETE | `/categories/{id}` | delete |
|
||||||
| GET | `/datasets` | paginated list of summaries (`?page=`, `?page_size=`, `?category_id=`) |
|
| GET | `/datasets` | paginated list of summaries (`?page=`, `?page_size=`, `?category_id=`, `?dg_id=`, `?start_date_from=`, `?start_date_to=`, `?end_date_from=`, `?end_date_to=`) |
|
||||||
| POST | `/datasets` | upload (multipart: `file`, `file_type`, `category_id`, `code`, `name`, `description?`, `unit?`, `meta?` (JSON), `automated?` (bool)) |
|
| POST | `/datasets` | upload (multipart: `file`, `file_type`, `category_id`, `code`, `name`, `description?`, `unit?`, `meta?` (JSON), `automated?` (bool), `dg_id?`, `start_date?`, `end_date?`) |
|
||||||
| GET | `/datasets/{id}` | full dataset (geometry as GeoJSON, `bbox` for rasters) |
|
| GET | `/datasets/{id}` | full dataset (geometry as GeoJSON, `bbox` for rasters) |
|
||||||
| GET | `/datasets/{id}.geojson` | GeoJSON `FeatureCollection`; plain `vector` returns its geometry as a single feature with the extracted attribute table as top-level properties; `vector_with_kato` maps observations, joining the `districts` table by KATO when it has no geometry of its own |
|
| GET | `/datasets/{id}.geojson` | GeoJSON `FeatureCollection`; plain `vector` returns its geometry as a single feature with the extracted attribute table as top-level properties; `vector_with_kato` always ignores its own geometry and joins the `districts` table by KATO, mapping observations onto each polygon |
|
||||||
| GET | `/datasets/{id}.kato.geojson` | GeoJSON `FeatureCollection` (vector_with_kato); ignores dataset geometry and always joins `districts` by KATO, mapping observations onto each polygon |
|
|
||||||
| GET | `/datasets/{id}/status` | processing status; long-polls with `?current=<status>` (holds up to `?wait=` secs, default 25, max 60) |
|
| GET | `/datasets/{id}/status` | processing status; long-polls with `?current=<status>` (holds up to `?wait=` secs, default 25, max 60) |
|
||||||
| GET | `/datasets/{id}/download` | download the stored file |
|
| GET | `/datasets/{id}/download` | download the stored file |
|
||||||
| POST | `/datasets/{id}/mapping` | set KATO column + year→date map (vector_with_kato) |
|
| POST | `/datasets/{id}/mapping` | set KATO column + year→period map (vector_with_kato) |
|
||||||
| GET | `/datasets/{id}/observations` | paginated unpivoted values (`?kato_code=`, `?page=`, `?page_size=`) |
|
| GET | `/datasets/{id}/observations` | paginated unpivoted values (`?kato_code=`, `?page=`, `?page_size=`) |
|
||||||
|
| POST | `/datasets/{id}/sld` | upload/replace the dataset's SLD (multipart `file`, `.sld`/`.xml`) |
|
||||||
|
| GET | `/datasets/{id}/sld` | download the dataset's SLD, falling back to its group's |
|
||||||
| DELETE | `/datasets/{id}` | delete (row + object) |
|
| DELETE | `/datasets/{id}` | delete (row + object) |
|
||||||
|
| GET | `/dataset-groups` | paginated list (`?code=`, `?page=`, `?page_size=`) |
|
||||||
|
| POST | `/dataset-groups` | create (`code`, `name`, `description?`) |
|
||||||
|
| GET | `/dataset-groups/{id}` | get |
|
||||||
|
| PUT | `/dataset-groups/{id}` | update |
|
||||||
|
| DELETE | `/dataset-groups/{id}` | delete (409 while datasets reference it) |
|
||||||
|
| GET | `/dataset-groups/{id}/latest` | freshest dataset of the group (`?status=`, `?file_type=`) |
|
||||||
|
| POST | `/dataset-groups/{id}/sld` | upload/replace the group's shared SLD |
|
||||||
|
| GET | `/dataset-groups/{id}/sld` | download the group's shared SLD |
|
||||||
|
| GET | `/landscape/slots` | ЛП input slots + the dataset filling each |
|
||||||
|
| POST | `/landscape/slots/{code}` | upload a GeoTIFF into a slot (multipart: `file`, `name?`, `start_date?`, `end_date?`) |
|
||||||
|
| GET | `/landscape/runs` | paginated runs, newest first (`?page=`, `?page_size=`) |
|
||||||
|
| POST | `/landscape/runs` | queue a run (`from_stage?`, `overrides?`, `params?`); 202 |
|
||||||
|
| GET | `/landscape/runs/{id}` | run with per-node status, inputs, outputs, timings |
|
||||||
|
| GET | `/landscape/results` | per output group: latest layer, `cog_url`, `sld_url` |
|
||||||
|
|
||||||
Example upload:
|
Example upload:
|
||||||
|
|
||||||
@@ -166,6 +365,17 @@ make lint # golangci-lint (if installed)
|
|||||||
CI (`.github/workflows/ci.yml`) runs build, vet, `go test -race`, and golangci-lint
|
CI (`.github/workflows/ci.yml`) runs build, vet, `go test -race`, and golangci-lint
|
||||||
on every push and pull request.
|
on every push and pull request.
|
||||||
|
|
||||||
|
Repository tests in `internal/repository/postgres` run against a real database
|
||||||
|
when `TEST_DB_URL` points at one with migrations applied, and are skipped
|
||||||
|
otherwise. They catch Postgres-only failures (parameter type deduction, casts,
|
||||||
|
constraints) that in-memory stubs cannot:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker run -d --name gis-test-pg -e POSTGRES_USER=gis -e POSTGRES_PASSWORD=gis -e POSTGRES_DB=gis -p 5434:5432 postgis/postgis:17-3.5
|
||||||
|
DB_URL='postgres://gis:gis@127.0.0.1:5434/gis?sslmode=disable' S3_ENDPOINT=x S3_ACCESS_KEY=x S3_SECRET_KEY=x RABBITMQ_URL=amqp://x go run ./cmd/gis migrate up
|
||||||
|
TEST_DB_URL='postgres://gis:gis@127.0.0.1:5434/gis?sslmode=disable' go test ./internal/repository/postgres/ -v
|
||||||
|
```
|
||||||
|
|
||||||
## Adding a feature
|
## Adding a feature
|
||||||
|
|
||||||
Each new domain is one vertical slice mirroring Category/Dataset:
|
Each new domain is one vertical slice mirroring Category/Dataset:
|
||||||
|
|||||||
950
api/openapi.yaml
950
api/openapi.yaml
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,9 @@
|
|||||||
# Build context is the repo root: docker build -f build/package/Dockerfile .
|
# Build context is the repo root: docker build -f build/package/Dockerfile .
|
||||||
FROM golang:1.26.1-alpine AS builder
|
FROM golang:1.26-alpine3.24 AS builder
|
||||||
|
|
||||||
|
# godal (GDAL bindings) needs cgo, a C/C++ toolchain and the GDAL headers.
|
||||||
|
# g++ is required: parts of the binding compile as C++ against GDAL's headers.
|
||||||
|
RUN apk add --no-cache gdal-dev gcc g++ musl-dev pkgconfig
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
@@ -7,13 +11,18 @@ COPY go.mod go.sum ./
|
|||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/gis ./cmd/gis
|
RUN CGO_ENABLED=1 GOOS=linux go build -trimpath -o /out/gis ./cmd/gis
|
||||||
|
|
||||||
FROM alpine:3.20
|
# Must stay on the same Alpine release as the builder — hence the pinned
|
||||||
|
# `golang:1.26-alpine3.24` above rather than the floating `-alpine` tag. A
|
||||||
|
# mismatch ships a different libgdal soname, and the image still builds green
|
||||||
|
# while the binary dies at startup with "libgdal.so.NN: No such file".
|
||||||
|
# Bump both lines together.
|
||||||
|
FROM alpine:3.24
|
||||||
|
|
||||||
# gdal-tools provides gdal_translate / gdalinfo for raster COG conversion and
|
# gdal: shared library for godal (raster math in the worker).
|
||||||
# ogr2ogr for extracting vector feature geometry (worker).
|
# gdal-tools: gdal_translate / gdalinfo (COG conversion) and ogr2ogr.
|
||||||
RUN apk add --no-cache ca-certificates tzdata gdal-tools \
|
RUN apk add --no-cache ca-certificates tzdata gdal gdal-tools \
|
||||||
&& adduser -D -u 10001 app
|
&& adduser -D -u 10001 app
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|||||||
@@ -28,3 +28,7 @@ RABBITMQ_QUEUE=gis.events.example
|
|||||||
|
|
||||||
# Host port mapping for the postgres container (docker-compose)
|
# Host port mapping for the postgres container (docker-compose)
|
||||||
DB_PORT=5432
|
DB_PORT=5432
|
||||||
|
|
||||||
|
# Landscape Program parallelism (bounds worker peak memory)
|
||||||
|
LP_MAX_WORKERS=8
|
||||||
|
LP_MAX_NODES=2
|
||||||
|
|||||||
2661
docs/superpowers/plans/2026-09-05-landscape-program.md
Normal file
2661
docs/superpowers/plans/2026-09-05-landscape-program.md
Normal file
File diff suppressed because it is too large
Load Diff
320
docs/superpowers/specs/2026-09-05-landscape-program-design.md
Normal file
320
docs/superpowers/specs/2026-09-05-landscape-program-design.md
Normal file
@@ -0,0 +1,320 @@
|
|||||||
|
# Landscape Program (ЛП) + dataset groups and dates — design
|
||||||
|
|
||||||
|
Date: 2026-09-05. Source: `LP_20260902.excalidraw`, `TZ_functional_zoning_go.md`,
|
||||||
|
`zoning_core.py`, `Landscape.ipynb` (reference numpy implementation).
|
||||||
|
Decisions taken with the product owner are recorded inline as **Decision**.
|
||||||
|
|
||||||
|
## 1. Goals
|
||||||
|
|
||||||
|
1. **Dataset versioning.** Datasets can belong to a *dataset group* ("одноимённые"
|
||||||
|
datasets), carry a `start_date` / `end_date` period, be filtered by those, and the
|
||||||
|
"freshest" dataset of a group can be fetched.
|
||||||
|
2. **Landscape Program pipeline.** A DAG of raster computations (Z, U, E, C, P_max,
|
||||||
|
C_max, functional zoning V1–V3) executed by the worker, with results stored as
|
||||||
|
ordinary raster datasets (COG + SLD) served by the existing dataset endpoints,
|
||||||
|
incremental recompute, and a dedicated upload for input rasters.
|
||||||
|
|
||||||
|
Non-goals: spatial operations (resampling, reprojection), GUI, changing the
|
||||||
|
vector_with_kato flow beyond the column rename below.
|
||||||
|
|
||||||
|
## 2. Part 1 — Dataset groups and dates
|
||||||
|
|
||||||
|
### 2.1 Schema (migration `00010_dataset_groups_and_dates.sql`)
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE dataset_groups (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
code VARCHAR(255) NOT NULL UNIQUE, -- slug, same rules as categories.code
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
description TEXT NOT NULL DEFAULT '',
|
||||||
|
sld_storage_key TEXT,
|
||||||
|
sld_filename VARCHAR(255),
|
||||||
|
sld_content_type VARCHAR(255),
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
ALTER TABLE datasets ADD COLUMN dg_id UUID REFERENCES dataset_groups(id) ON DELETE RESTRICT;
|
||||||
|
ALTER TABLE datasets ADD COLUMN start_date DATE;
|
||||||
|
ALTER TABLE datasets ADD COLUMN end_date DATE;
|
||||||
|
CREATE INDEX idx_datasets_dg_id_end_date ON datasets (dg_id, end_date DESC NULLS LAST, created_at DESC);
|
||||||
|
CREATE INDEX idx_datasets_start_date ON datasets (start_date);
|
||||||
|
CREATE INDEX idx_datasets_end_date ON datasets (end_date);
|
||||||
|
|
||||||
|
-- Decision: rename the observation/mapping period columns (breaking API change).
|
||||||
|
ALTER TABLE dataset_observations RENAME COLUMN date TO start_date;
|
||||||
|
ALTER TABLE dataset_observations RENAME COLUMN date_to TO end_date;
|
||||||
|
-- unique index and idx_dataset_observations_kato_date recreated on the new names.
|
||||||
|
-- datasets.year_columns JSONB: every element's keys "date" -> "start_date",
|
||||||
|
-- "date_to" -> "end_date" (UPDATE with jsonb_agg over jsonb_array_elements; NULL-safe).
|
||||||
|
```
|
||||||
|
|
||||||
|
Constraint: `end_date >= start_date` when both set (CHECK constraint + service validation).
|
||||||
|
|
||||||
|
### 2.2 Domain changes
|
||||||
|
|
||||||
|
- `domain.DatasetGroup{ID, Code, Name, Description, SldStorageKey, SldFilename,
|
||||||
|
SldContentType, CreatedAt, UpdatedAt}`.
|
||||||
|
- `domain.Dataset` and `domain.DatasetSummary` gain `DatasetGroupID *uuid.UUID`
|
||||||
|
(`json:"dg_id"`), `StartDate *string` (`json:"start_date"`, `YYYY-MM-DD`),
|
||||||
|
`EndDate *string` (`json:"end_date"`).
|
||||||
|
- `domain.Observation`: `Date`→`StartDate` (`json:"start_date"`), `DateTo`→`EndDate`
|
||||||
|
(`json:"end_date"`). `domain.YearColumn`: same rename. Mapping request body uses
|
||||||
|
`start_date` (required) / `end_date` (optional). Everything that previously said
|
||||||
|
`date`/`date_to` in code, OpenAPI and README is renamed. GeoJSON output keeps its
|
||||||
|
`data` (keyed by start date) and `periods` (start→end) shape.
|
||||||
|
- `domain.DatasetFilter` gains `DatasetGroupID *uuid.UUID`, `StartDateFrom`,
|
||||||
|
`StartDateTo`, `EndDateFrom`, `EndDateTo *string` (inclusive bounds).
|
||||||
|
|
||||||
|
### 2.3 API
|
||||||
|
|
||||||
|
| Method & path | Behaviour |
|
||||||
|
|---|---|
|
||||||
|
| `GET /dataset-groups` | paginated list, `?code=` exact filter optional |
|
||||||
|
| `POST /dataset-groups` | `{code, name, description}`; 201 |
|
||||||
|
| `GET /dataset-groups/{id}` | group |
|
||||||
|
| `PUT /dataset-groups/{id}` | replace `{code, name, description}` |
|
||||||
|
| `DELETE /dataset-groups/{id}` | 204; 409 when datasets reference it; also removes group SLD object |
|
||||||
|
| `GET /dataset-groups/{id}/latest` | freshest dataset of the group: order by `end_date DESC NULLS LAST, created_at DESC`; optional `?status=`, `?file_type=`; 404 when none |
|
||||||
|
| `POST /dataset-groups/{id}/sld` | multipart `file` (`.sld`/`.xml`), replaces; mirrors dataset SLD upload |
|
||||||
|
| `GET /dataset-groups/{id}/sld` | download; 404 when none |
|
||||||
|
| `GET /datasets` | new filters `dg_id`, `start_date_from`, `start_date_to`, `end_date_from`, `end_date_to` (all `YYYY-MM-DD`, inclusive, any combination) |
|
||||||
|
| `POST /datasets` | new optional form fields `dg_id`, `start_date`, `end_date` |
|
||||||
|
| `PUT /datasets/{id}` | body gains `dg_id`, `start_date`, `end_date` (nullable; omitted/null clears) |
|
||||||
|
| `PATCH /datasets/{id}` | sparse; same three fields, `null` clears |
|
||||||
|
| `GET /datasets/{id}/sld` | **fallback**: when the dataset has no SLD but its group has one, the group SLD is served |
|
||||||
|
|
||||||
|
Validation: `dg_id` must reference an existing group (404/422 as for `category_id`);
|
||||||
|
dates parse as `2006-01-02`; `end_date` not before `start_date`.
|
||||||
|
|
||||||
|
`DatasetSummary` (list responses) includes `dg_id`, `start_date`, `end_date`.
|
||||||
|
|
||||||
|
### 2.4 Freshness rule
|
||||||
|
|
||||||
|
"Самый свежий" = greatest `end_date`; datasets with `NULL end_date` sort after all
|
||||||
|
dated ones; ties broken by `created_at DESC`. The same rule is used by the pipeline
|
||||||
|
to resolve inputs (with `status = ready`, `file_type = raster`).
|
||||||
|
|
||||||
|
## 3. Part 2 — Landscape Program
|
||||||
|
|
||||||
|
### 3.1 Computation graph
|
||||||
|
|
||||||
|
Node code `XY`: `X` = stage, `Y` = function in the stage. A stage starts only when
|
||||||
|
every node of all lower stages finished. Every input and output of every node is a
|
||||||
|
**dataset group**; leaf groups are the *slots* users upload to.
|
||||||
|
|
||||||
|
| Node | Name | Inputs (group codes) | Outputs (group codes) |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 00 | z_integrated | `lp-z-biodiversity`, `lp-z-lcr`, `lp-z-soil-protection`, `lp-z-wr` | `lp-z-integrated` |
|
||||||
|
| 01 | u_integrated | `lp-u-surf-water`, `lp-u-soil`, `lp-u-crop`, `lp-u-veg` | `lp-u-integrated` |
|
||||||
|
| 02 | z_max | Z1..Z4 as above | `lp-z-max` |
|
||||||
|
| 03 | u_max | U as above | `lp-u-max` |
|
||||||
|
| 10 | e_ecological_constraint | `lp-z-integrated`, `lp-u-integrated` | `lp-e` |
|
||||||
|
| 20 | c_conflicts | `lp-p-irrigation`, `lp-p-snow`, `lp-p-crop`, `lp-p-pasture`, `lp-p-tourism`, `lp-e` | `lp-c-irrigation`, `lp-c-snow`, `lp-c-crop`, `lp-c-pasture`, `lp-c-tourism` |
|
||||||
|
| 21 | p_max | five P groups | `lp-p-max` |
|
||||||
|
| 30 | func_zoning | `lp-e`, five P groups (original suitability) | `lp-v1-conservative`, `lp-v2-balanced`, `lp-v3-development` |
|
||||||
|
| 31 | c_max | five C groups | `lp-c-max` |
|
||||||
|
|
||||||
|
Sector order everywhere: 1 irrigation, 2 snow, 3 crop, 4 pasture, 5 tourism.
|
||||||
|
**Decision (revised 2026-09-11):** func_zoning takes S_k = P_k, the original suitability
|
||||||
|
rasters, as in the reference notebook ("Three scenarios" cells) and `zoning_core.py`.
|
||||||
|
The diagram folder `300-…-Equals_102_202` pointing at the C layers was a drawing error;
|
||||||
|
C_k = min(P_k, E) are diagnostic conflict layers and never replace P_k in zoning.
|
||||||
|
|
||||||
|
Groups and the category `landscape` ("Ландшафтная программа") are seeded by
|
||||||
|
migration `00011_landscape.sql` (`INSERT … ON CONFLICT (code) DO NOTHING`). The
|
||||||
|
graph itself lives in Go (`internal/landscape/graph.go`), not in the DB.
|
||||||
|
|
||||||
|
### 3.2 Pixel rules (from the reference notebook; must match it bitwise)
|
||||||
|
|
||||||
|
Notation: a value is *valid* iff `1 <= v <= 5`; anything else (including nodata) is
|
||||||
|
invalid. Output nodata is `-9999`. `classify(x)` maps a float32 score to a class:
|
||||||
|
`x <= 1.8 → 1`, `<= 2.6 → 2`, `<= 3.4 → 3`, `<= 4.2 → 4`, `<= 5.0 → 5` (and requires
|
||||||
|
`x >= 1.0`); comparisons are done in float32 against float32 constants, exactly as
|
||||||
|
numpy does with a float32 array.
|
||||||
|
|
||||||
|
| Node | Valid domain (pixel) | Value |
|
||||||
|
|---|---|---|
|
||||||
|
| z_integrated | all four Z valid | `classify(0.25*Z1 + 0.25*Z2 + 0.25*Z3 + 0.25*Z4)` (float32) |
|
||||||
|
| u_integrated | water ∧ soil ∧ (crop ∨ veg) | `classify(num/den)`, `num = 0.25*water + 0.25*soil + [0.25*crop if crop valid] + [0.25*veg if veg valid]`, `den = 0.5 + [0.25] + [0.25]` |
|
||||||
|
| z_max | all four valid | `max(Z1..Z4)` |
|
||||||
|
| u_max | water ∧ soil ∧ (crop ∨ veg) | `max` over the valid ones of water, soil, crop, veg |
|
||||||
|
| e | Z ∧ U valid | `max(Z, U)` |
|
||||||
|
| c_conflicts (5 outputs) | P_j ∧ E valid | `min(P_j, E)` |
|
||||||
|
| p_max | at least one P valid | `max` over valid P |
|
||||||
|
| c_max | at least one C valid | `max` over valid C |
|
||||||
|
| func_zoning (3 outputs) | see TZ §4; S_k = P_k | see below |
|
||||||
|
|
||||||
|
Functional zoning (TZ `TZ_functional_zoning_go.md`, reference `zoning_core.py`):
|
||||||
|
|
||||||
|
```
|
||||||
|
eValid = valid(E); base_k = eValid && valid(S_k); analysisValid = eValid && any(base_k)
|
||||||
|
if !analysisValid → -9999 in all scenarios
|
||||||
|
candidate_k = base_k && rule_v(E, S_k)
|
||||||
|
V1: (E <= 2 && S >= v1_min_s_e12) || (E == 3 && S >= v1_min_s_e3) (defaults 3, 4)
|
||||||
|
V2: (E <= 2) || (E == 3 && S >= v2_min_s_e3) || (E == 4 && S >= v2_min_s_e4) (defaults 3, 5)
|
||||||
|
V3: (E <= 4)
|
||||||
|
no candidates → class 1 (regulated); else winners = candidates with S == max(S over candidates);
|
||||||
|
sector = first of tie_priority (default [1,2,3,4,5]) among winners; class = sector + 1
|
||||||
|
```
|
||||||
|
|
||||||
|
Output classes: 1 regulated, 2 irrigation, 3 snow, 4 crop, 5 pasture, 6 tourism.
|
||||||
|
|
||||||
|
### 3.3 Raster I/O abstraction (**Decision:** godal behind an interface)
|
||||||
|
|
||||||
|
Package `internal/raster` (existing) gains an engine-agnostic interface so the
|
||||||
|
implementation can be swapped (godal today; `gdal_calc`, ENVI+CLI or another binding
|
||||||
|
later) without touching the pipeline:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type Grid struct { Width, Height int; GeoTransform [6]float64; ProjectionWKT string }
|
||||||
|
type Window struct { X, Y, Width, Height int }
|
||||||
|
type Reader interface {
|
||||||
|
Grid() Grid
|
||||||
|
NoData() (value float64, ok bool)
|
||||||
|
ReadFloat64(w Window, dst []float64) error // band 1, converted by the driver
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
type WriterOptions struct { NoData int16; TileSize int; Compression string /* DEFLATE|ZSTD */ }
|
||||||
|
type Writer interface {
|
||||||
|
WriteInt16(w Window, src []int16) error // band 1
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
type IO interface {
|
||||||
|
Open(ctx context.Context, path string) (Reader, error)
|
||||||
|
Create(ctx context.Context, path string, g Grid, o WriterOptions) (Writer, error)
|
||||||
|
SameGrid(a, b Grid) bool // size, transform (abs tol 1e-9), CRS (GDAL IsSame)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`internal/raster/godal.go` implements `IO` with `github.com/airbusgeo/godal` (cgo).
|
||||||
|
Output GeoTIFF: `int16`, nodata `-9999`, `TILED=YES`, `BLOCKXSIZE=BLOCKYSIZE=512`,
|
||||||
|
`COMPRESS=DEFLATE`, `PREDICTOR=2`, `BIGTIFF=IF_SAFER`; georeference copied from the
|
||||||
|
node's first input (for func_zoning: E). Inputs are read as float64 regardless of
|
||||||
|
their storage type; multi-band inputs use band 1.
|
||||||
|
|
||||||
|
The existing `RasterConverter` (gdal_translate / gdalinfo / ogr2ogr CLI) stays as is
|
||||||
|
and keeps producing the COG for every result dataset.
|
||||||
|
|
||||||
|
Build: `CGO_ENABLED=1`; `go.mod` gets `toolchain go1.26.2` (the local SDK is
|
||||||
|
incomplete) and the godal dependency. Dockerfile: builder `golang:1.26-alpine` +
|
||||||
|
`gdal-dev gcc musl-dev pkgconfig`; runtime `alpine:3.22` + `gdal gdal-tools`.
|
||||||
|
|
||||||
|
### 3.4 Compute engine (`internal/landscape`)
|
||||||
|
|
||||||
|
- `graph.go`: slots, nodes, stages, group codes, node-level param subsets.
|
||||||
|
- `kernels.go`: pure per-window functions over `[]float64` inputs → `[]int16`
|
||||||
|
outputs, one per node. No I/O, fully unit-tested (TZ §7 edge cases included).
|
||||||
|
- `compute.go`: `Compute(ctx, io raster.IO, node Node, inputs map[string]string
|
||||||
|
/*group→local path*/, outDir string, p Params) (map[string]string /*group→path*/, error)`.
|
||||||
|
Opens inputs, **fails if grids differ** (**Decision:** no resampling), creates
|
||||||
|
outputs on the first input's grid, iterates windows of `p.WindowSize`
|
||||||
|
(default 2048) with a worker pool: each worker owns its own `Reader` handles
|
||||||
|
(GDAL datasets are not goroutine-safe); writes are serialised per output through a
|
||||||
|
mutex. Window order does not affect results.
|
||||||
|
- Nodes of one stage run concurrently (`errgroup`); stages run sequentially.
|
||||||
|
|
||||||
|
### 3.5 Runs, incremental recompute
|
||||||
|
|
||||||
|
Migration `00011_landscape.sql`:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE lp_runs (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
status VARCHAR(32) NOT NULL, -- queued | running | succeeded | failed
|
||||||
|
from_stage SMALLINT, -- NULL = incremental
|
||||||
|
params JSONB NOT NULL,
|
||||||
|
overrides JSONB NOT NULL DEFAULT '{}', -- {group_code: dataset_id}
|
||||||
|
error TEXT,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
started_at TIMESTAMPTZ,
|
||||||
|
finished_at TIMESTAMPTZ
|
||||||
|
);
|
||||||
|
CREATE TABLE lp_run_nodes (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
run_id UUID NOT NULL REFERENCES lp_runs(id) ON DELETE CASCADE,
|
||||||
|
node_code VARCHAR(8) NOT NULL,
|
||||||
|
stage SMALLINT NOT NULL,
|
||||||
|
status VARCHAR(32) NOT NULL, -- pending | running | skipped | succeeded | failed
|
||||||
|
fingerprint TEXT NOT NULL,
|
||||||
|
input_datasets JSONB NOT NULL, -- [{group, dataset_id}]
|
||||||
|
output_datasets JSONB, -- [{group, dataset_id}]
|
||||||
|
error TEXT,
|
||||||
|
started_at TIMESTAMPTZ,
|
||||||
|
finished_at TIMESTAMPTZ,
|
||||||
|
UNIQUE (run_id, node_code)
|
||||||
|
);
|
||||||
|
CREATE INDEX idx_lp_run_nodes_lookup ON lp_run_nodes (node_code, fingerprint, status);
|
||||||
|
```
|
||||||
|
|
||||||
|
Run execution (worker, queue `gis.landscape.run`, routing key `landscape.run`):
|
||||||
|
|
||||||
|
1. Mark run `running`. For each stage in order, for each node concurrently:
|
||||||
|
2. **Resolve inputs** per input group: `overrides[group]` if present; else the dataset
|
||||||
|
produced for that group earlier in *this* run; else the freshest `ready` raster of
|
||||||
|
the group (§2.4). Missing → node fails with a clear message.
|
||||||
|
3. **Fingerprint** = sha256 of node code, sorted `(group, dataset_id)` pairs, and the
|
||||||
|
node-relevant params. If `from_stage` is NULL or `stage < from_stage`, and a
|
||||||
|
previous `succeeded` node row with the same fingerprint exists whose output
|
||||||
|
datasets all still exist → mark `skipped`, reuse those outputs. Otherwise compute.
|
||||||
|
4. Download inputs to a temp dir, `Compute`, then for each output create a raster
|
||||||
|
dataset (§3.6), record `output_datasets`, mark `succeeded`.
|
||||||
|
5. Any node failure fails the stage; remaining stages are not started; run `failed`
|
||||||
|
with the error. Otherwise run `succeeded`.
|
||||||
|
|
||||||
|
`from_stage = 0` recomputes everything ("триггер на перерасчёт всего").
|
||||||
|
|
||||||
|
### 3.6 Result datasets
|
||||||
|
|
||||||
|
Every output is stored through a new service method
|
||||||
|
`DatasetService.CreateRasterFromFile(ctx, CreateRasterInput{CategoryID, GroupID,
|
||||||
|
Name, Filename, Path, StartDate, EndDate, Meta})`: uploads the file, inserts a
|
||||||
|
`raster` dataset (`automated = true`, `status = processing`), enqueues the existing
|
||||||
|
COG conversion. `Name` = `<Output name> <run started_at date>`, `Filename` =
|
||||||
|
`<group>.tif`, `start_date`/`end_date` = min/max over the node's inputs (NULL if all
|
||||||
|
NULL), `meta = {"lp_run_id", "lp_node", "inputs": [{group, dataset_id}]}`.
|
||||||
|
Downstream nodes read the original object (`storage_key`), not the COG, so they never
|
||||||
|
wait for conversion. SLD for a result comes from its group (dataset SLD fallback, §2.3).
|
||||||
|
|
||||||
|
### 3.7 Landscape API (`/landscape`)
|
||||||
|
|
||||||
|
| Method & path | Behaviour |
|
||||||
|
|---|---|
|
||||||
|
| `GET /landscape/slots` | every leaf slot: `code`, `title`, group id, latest dataset summary (any status) or null |
|
||||||
|
| `POST /landscape/slots/{code}` | **dedicated input upload**: multipart `file` (`.tif/.tiff`, TIFF magic check), optional `name`, `start_date`, `end_date`. Creates a `raster` dataset in the slot's group under the `landscape` category via the regular upload path (COG conversion enqueued). 201 Dataset. Unknown slot → 404 |
|
||||||
|
| `POST /landscape/runs` | body `{from_stage?: 0..3, overrides?: {group_code: dataset_id}, params?: {window_size?, tie_priority?, v1_min_s_e12?, v1_min_s_e3?, v2_min_s_e3?, v2_min_s_e4?}}`; validates overrides (existing ready raster), inserts run `queued` + pending node rows, publishes job; 202 with run |
|
||||||
|
| `GET /landscape/runs` | paginated, newest first |
|
||||||
|
| `GET /landscape/runs/{id}` | run with nodes (status, inputs, outputs, error, timings) |
|
||||||
|
| `GET /landscape/results` | per output group: `group`, `title`, `node`, latest `ready` dataset (or null) with `cog_url = /datasets/{id}.cog`, `sld_url` when the group has an SLD |
|
||||||
|
|
||||||
|
Params defaults: `window_size 2048`, `tie_priority [1,2,3,4,5]`, `v1 3/4`, `v2 3/5`.
|
||||||
|
Validation: tie_priority is a permutation of 1..5; thresholds in 1..5; window_size
|
||||||
|
256..8192.
|
||||||
|
|
||||||
|
### 3.8 Error handling
|
||||||
|
|
||||||
|
- Grid mismatch, invalid CRS, missing input, GDAL error → node `failed` with message;
|
||||||
|
run `failed`. No retry (consistent with existing consumers: nack without requeue).
|
||||||
|
- Non-nodata values outside 1..5 are treated as invalid pixels (TZ §2), not fatal.
|
||||||
|
- Result dataset creation failure after compute → node failed, temp files removed.
|
||||||
|
- Temp files always cleaned with `defer`.
|
||||||
|
|
||||||
|
### 3.9 Testing
|
||||||
|
|
||||||
|
- Kernels: table tests incl. TZ §7 cases (invalid E, all S invalid, E=5, E=4 with
|
||||||
|
several S=5, all S equal, applicability of crop/veg for U, "any valid" for max nodes).
|
||||||
|
- `Compute`: tests with an in-memory fake `raster.IO` (10×10 grids), including grid
|
||||||
|
mismatch failure and window tiling with sizes that do not divide the raster.
|
||||||
|
- godal implementation: round-trip test on a temp GeoTIFF (runs when GDAL present).
|
||||||
|
- Services: stub repositories as in `dataset_test.go`; run planning (fingerprint,
|
||||||
|
skip/reuse, from_stage), input resolution order (override → this run → latest).
|
||||||
|
- HTTP: handler tests for validation paths.
|
||||||
|
- Part 1: filter clause building, latest ordering (nulls last), rename in mapping and
|
||||||
|
observations, SLD fallback.
|
||||||
|
|
||||||
|
### 3.10 Out of scope / follow-ups
|
||||||
|
|
||||||
|
- Resampling/alignment of inputs (fails by design).
|
||||||
|
- Continuous (float) intermediate rasters and diagnostic layers from the notebook
|
||||||
|
(tie counts, component counts, critical flags).
|
||||||
|
- Real-data acceptance run against the Drive test set (inputs there are not aligned).
|
||||||
3
go.mod
3
go.mod
@@ -2,7 +2,10 @@ module gis
|
|||||||
|
|
||||||
go 1.26.1
|
go 1.26.1
|
||||||
|
|
||||||
|
toolchain go1.26.2
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/airbusgeo/godal v0.0.18
|
||||||
github.com/caarlos0/env/v11 v11.4.1
|
github.com/caarlos0/env/v11 v11.4.1
|
||||||
github.com/go-chi/chi/v5 v5.3.0
|
github.com/go-chi/chi/v5 v5.3.0
|
||||||
github.com/go-playground/validator/v10 v10.30.3
|
github.com/go-playground/validator/v10 v10.30.3
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -1,3 +1,5 @@
|
|||||||
|
github.com/airbusgeo/godal v0.0.18 h1:NQJ9P7OgSs5zVtlWH/tOta7QHYbbtha7Nttf63YM01Q=
|
||||||
|
github.com/airbusgeo/godal v0.0.18/go.mod h1:Mpn/6exBzEbIx4DaG2rIjCmAiF9Ww4SJN2ZtwVsSwCo=
|
||||||
github.com/caarlos0/env/v11 v11.4.1 h1:fYwH0sWEsBSMPG7t4e/PEfTFzrWrpjyygXyUnWiSwEw=
|
github.com/caarlos0/env/v11 v11.4.1 h1:fYwH0sWEsBSMPG7t4e/PEfTFzrWrpjyygXyUnWiSwEw=
|
||||||
github.com/caarlos0/env/v11 v11.4.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
|
github.com/caarlos0/env/v11 v11.4.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
|
|||||||
@@ -38,6 +38,8 @@ type App struct {
|
|||||||
publisher *rabbitmq.Publisher
|
publisher *rabbitmq.Publisher
|
||||||
categories *service.CategoryService
|
categories *service.CategoryService
|
||||||
datasets *service.DatasetService
|
datasets *service.DatasetService
|
||||||
|
groups *service.DatasetGroupService
|
||||||
|
landscape *service.LandscapeService
|
||||||
eventRepo *postgres.EventRepository
|
eventRepo *postgres.EventRepository
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,10 +70,16 @@ func New(ctx context.Context) (*App, error) {
|
|||||||
|
|
||||||
categoryRepo := postgres.NewCategoryRepository(pool)
|
categoryRepo := postgres.NewCategoryRepository(pool)
|
||||||
datasetRepo := postgres.NewDatasetRepository(pool)
|
datasetRepo := postgres.NewDatasetRepository(pool)
|
||||||
|
groupRepo := postgres.NewDatasetGroupRepository(pool)
|
||||||
eventRepo := postgres.NewEventRepository(pool)
|
eventRepo := postgres.NewEventRepository(pool)
|
||||||
|
|
||||||
|
lpRepo := postgres.NewLandscapeRepository(pool)
|
||||||
|
|
||||||
publisher := rabbitmq.NewPublisher(rabbit)
|
publisher := rabbitmq.NewPublisher(rabbit)
|
||||||
jobPublisher := rabbitmq.NewDatasetJobPublisher(publisher)
|
jobPublisher := rabbitmq.NewDatasetJobPublisher(publisher)
|
||||||
|
lpJobs := rabbitmq.NewLandscapeJobPublisher(publisher)
|
||||||
|
|
||||||
|
datasetsSvc := service.NewDatasetService(datasetRepo, store, categoryRepo, groupRepo, jobPublisher, parser.Columns, parser.Rows, raster.NewGDALConverter())
|
||||||
|
|
||||||
return &App{
|
return &App{
|
||||||
Cfg: cfg,
|
Cfg: cfg,
|
||||||
@@ -81,8 +89,19 @@ func New(ctx context.Context) (*App, error) {
|
|||||||
rabbit: rabbit,
|
rabbit: rabbit,
|
||||||
publisher: publisher,
|
publisher: publisher,
|
||||||
categories: service.NewCategoryService(categoryRepo),
|
categories: service.NewCategoryService(categoryRepo),
|
||||||
datasets: service.NewDatasetService(datasetRepo, store, categoryRepo, jobPublisher, parser.Columns, parser.Rows, raster.NewGDALConverter()),
|
datasets: datasetsSvc,
|
||||||
eventRepo: eventRepo,
|
groups: service.NewDatasetGroupService(groupRepo, datasetRepo, store),
|
||||||
|
// A nil ComputeFunc selects landscape.ComputeWithWorkers, so each node
|
||||||
|
// gets the per-node share of the configured worker budget.
|
||||||
|
landscape: service.NewLandscapeService(
|
||||||
|
lpRepo, datasetRepo, groupRepo, categoryRepo, datasetsSvc, store,
|
||||||
|
lpJobs, raster.NewGodalIO(), nil, log,
|
||||||
|
service.LandscapeOptions{
|
||||||
|
MaxWorkers: cfg.Landscape.MaxWorkers,
|
||||||
|
MaxConcurrentNodes: cfg.Landscape.MaxConcurrentNodes,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
eventRepo: eventRepo,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,11 +119,13 @@ func (a *App) Handler() stdhttp.Handler {
|
|||||||
})
|
})
|
||||||
|
|
||||||
return transporthttp.NewRouter(transporthttp.RouterDeps{
|
return transporthttp.NewRouter(transporthttp.RouterDeps{
|
||||||
Logger: a.Log,
|
Logger: a.Log,
|
||||||
Health: health,
|
Health: health,
|
||||||
Categories: transporthttp.NewCategoryHandler(a.categories, validate),
|
Categories: transporthttp.NewCategoryHandler(a.categories, validate),
|
||||||
Datasets: transporthttp.NewDatasetHandler(a.datasets, validate),
|
Datasets: transporthttp.NewDatasetHandler(a.datasets, validate),
|
||||||
OpenAPISpec: api.Spec,
|
DatasetGroups: transporthttp.NewDatasetGroupHandler(a.groups, validate),
|
||||||
|
Landscape: transporthttp.NewLandscapeHandler(a.landscape, validate),
|
||||||
|
OpenAPISpec: api.Spec,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,10 +141,20 @@ func (a *App) Consumers() []*rabbitmq.Consumer {
|
|||||||
a.PropertiesConsumer(),
|
a.PropertiesConsumer(),
|
||||||
a.ExtractConsumer(),
|
a.ExtractConsumer(),
|
||||||
a.ConvertConsumer(),
|
a.ConvertConsumer(),
|
||||||
|
a.LandscapeConsumer(),
|
||||||
a.ExampleConsumer(),
|
a.ExampleConsumer(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LandscapeConsumer builds the Landscape Program run consumer.
|
||||||
|
func (a *App) LandscapeConsumer() *rabbitmq.Consumer {
|
||||||
|
handler := rabbitmq.NewLandscapeRunHandler(a.landscape, a.Log)
|
||||||
|
return rabbitmq.NewConsumer(
|
||||||
|
a.rabbit, rabbitmq.LandscapeRunQueue, rabbitmq.LandscapeRunRoutingKey,
|
||||||
|
"gis-landscape-runner", handler, a.Log,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// PropertiesConsumer builds the plain-vector properties-extraction consumer.
|
// PropertiesConsumer builds the plain-vector properties-extraction consumer.
|
||||||
func (a *App) PropertiesConsumer() *rabbitmq.Consumer {
|
func (a *App) PropertiesConsumer() *rabbitmq.Consumer {
|
||||||
handler := rabbitmq.NewPropertiesHandler(a.datasets, a.Log)
|
handler := rabbitmq.NewPropertiesHandler(a.datasets, a.Log)
|
||||||
@@ -178,11 +209,12 @@ func (a *App) ReprocessDataset(ctx context.Context, id uuid.UUID) (domain.Datase
|
|||||||
return a.datasets.Reprocess(ctx, id)
|
return a.datasets.Reprocess(ctx, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReprocessAllDatasets re-enqueues the processing job for every dataset,
|
// ReprocessAllDatasets re-enqueues the processing job for every dataset
|
||||||
// restarting each one's asynchronous pipeline. It returns how many jobs were
|
// matching fileType (nil matches every file type), restarting each one's
|
||||||
// enqueued and a per-dataset map of any failures.
|
// asynchronous pipeline. It returns how many jobs were enqueued and a
|
||||||
func (a *App) ReprocessAllDatasets(ctx context.Context) (int, map[uuid.UUID]error, error) {
|
// per-dataset map of any failures.
|
||||||
return a.datasets.ReprocessAll(ctx)
|
func (a *App) ReprocessAllDatasets(ctx context.Context, fileType *domain.FileType) (int, map[uuid.UUID]error, error) {
|
||||||
|
return a.datasets.ReprocessAll(ctx, fileType)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close releases all resources in reverse order of acquisition.
|
// Close releases all resources in reverse order of acquisition.
|
||||||
|
|||||||
@@ -2,31 +2,53 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"gis/internal/app"
|
"gis/internal/app"
|
||||||
|
"gis/internal/domain"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
var reprocessAll bool
|
var (
|
||||||
|
reprocessAll bool
|
||||||
|
reprocessType string
|
||||||
|
)
|
||||||
|
|
||||||
var reprocessCmd = &cobra.Command{
|
var reprocessCmd = &cobra.Command{
|
||||||
Use: "reprocess [dataset-id]",
|
Use: "reprocess [dataset-id]",
|
||||||
Short: "Re-enqueue the processing job for uploaded datasets",
|
Short: "Re-enqueue the processing job for uploaded datasets",
|
||||||
Long: "Re-publish the RabbitMQ message that drives an uploaded dataset's\n" +
|
Long: "Re-publish the RabbitMQ message that drives an uploaded dataset's\n" +
|
||||||
"asynchronous processing, selecting the right step from its file type\n" +
|
"asynchronous processing, selecting the right step from its file type\n" +
|
||||||
"(vector_with_kato -> parse, vector -> properties, raster -> cog).\n\n" +
|
"(vector_with_kato -> parse, vector -> properties, raster -> cog).\n" +
|
||||||
|
"Before re-enqueuing, each dataset's stored geometry and GeoJSON are\n" +
|
||||||
|
"cleared so a stale (e.g. previously miscomputed) value is never served\n" +
|
||||||
|
"while the job is pending.\n\n" +
|
||||||
"Pass a dataset id to reprocess a single dataset, or --all to reprocess\n" +
|
"Pass a dataset id to reprocess a single dataset, or --all to reprocess\n" +
|
||||||
"every dataset.\n\n" +
|
"every dataset. --type restricts --all to one file type\n" +
|
||||||
|
"(vector_with_kato, vector, or raster).\n\n" +
|
||||||
"Examples:\n" +
|
"Examples:\n" +
|
||||||
" gis reprocess 06818b2b-1fc5-47d9-a764-db2d4cb3df75\n" +
|
" gis reprocess 06818b2b-1fc5-47d9-a764-db2d4cb3df75\n" +
|
||||||
" gis reprocess --all",
|
" gis reprocess --all\n" +
|
||||||
|
" gis reprocess --all --type=vector",
|
||||||
Args: cobra.MaximumNArgs(1),
|
Args: cobra.MaximumNArgs(1),
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
if reprocessAll == (len(args) == 1) {
|
if reprocessAll == (len(args) == 1) {
|
||||||
return fmt.Errorf("provide exactly one of <dataset-id> or --all")
|
return fmt.Errorf("provide exactly one of <dataset-id> or --all")
|
||||||
}
|
}
|
||||||
|
if reprocessType != "" && !reprocessAll {
|
||||||
|
return fmt.Errorf("--type can only be used together with --all")
|
||||||
|
}
|
||||||
|
|
||||||
|
var fileType *domain.FileType
|
||||||
|
if reprocessType != "" {
|
||||||
|
ft := domain.FileType(strings.TrimSpace(reprocessType))
|
||||||
|
if !ft.Valid() {
|
||||||
|
return fmt.Errorf("invalid --type %q: must be one of vector_with_kato, vector, raster", reprocessType)
|
||||||
|
}
|
||||||
|
fileType = &ft
|
||||||
|
}
|
||||||
|
|
||||||
ctx, cancel := signalContext()
|
ctx, cancel := signalContext()
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -38,7 +60,7 @@ var reprocessCmd = &cobra.Command{
|
|||||||
defer application.Close()
|
defer application.Close()
|
||||||
|
|
||||||
if reprocessAll {
|
if reprocessAll {
|
||||||
enqueued, failures, err := application.ReprocessAllDatasets(ctx)
|
enqueued, failures, err := application.ReprocessAllDatasets(ctx, fileType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -47,7 +69,7 @@ var reprocessCmd = &cobra.Command{
|
|||||||
"dataset_id", id, "error", ferr)
|
"dataset_id", id, "error", ferr)
|
||||||
}
|
}
|
||||||
application.Log.Info("re-enqueued all dataset processing",
|
application.Log.Info("re-enqueued all dataset processing",
|
||||||
"enqueued", enqueued, "failed", len(failures))
|
"file_type", reprocessType, "enqueued", enqueued, "failed", len(failures))
|
||||||
if len(failures) > 0 {
|
if len(failures) > 0 {
|
||||||
return fmt.Errorf("%d of %d datasets failed to re-enqueue", len(failures), enqueued+len(failures))
|
return fmt.Errorf("%d of %d datasets failed to re-enqueue", len(failures), enqueued+len(failures))
|
||||||
}
|
}
|
||||||
@@ -71,4 +93,5 @@ var reprocessCmd = &cobra.Command{
|
|||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
reprocessCmd.Flags().BoolVar(&reprocessAll, "all", false, "reprocess every dataset")
|
reprocessCmd.Flags().BoolVar(&reprocessAll, "all", false, "reprocess every dataset")
|
||||||
|
reprocessCmd.Flags().StringVar(&reprocessType, "type", "", "restrict --all to one file type (vector_with_kato, vector, raster)")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ var publishExample bool
|
|||||||
|
|
||||||
var workerCmd = &cobra.Command{
|
var workerCmd = &cobra.Command{
|
||||||
Use: "worker",
|
Use: "worker",
|
||||||
Short: "Run the RabbitMQ consumers (dataset parsing + example)",
|
Short: "Run the RabbitMQ consumers (dataset processing, landscape runs + example)",
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
ctx, cancel := signalContext()
|
ctx, cancel := signalContext()
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|||||||
@@ -12,10 +12,11 @@ import (
|
|||||||
// Config holds all configuration for the application. Values are read from
|
// Config holds all configuration for the application. Values are read from
|
||||||
// environment variables; a local .env file (if present) is loaded first.
|
// environment variables; a local .env file (if present) is loaded first.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
HTTP HTTPConfig
|
HTTP HTTPConfig
|
||||||
DB DBConfig
|
DB DBConfig
|
||||||
S3 S3Config
|
S3 S3Config
|
||||||
RabbitMQ RabbitMQConfig
|
RabbitMQ RabbitMQConfig
|
||||||
|
Landscape LandscapeConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
// HTTPConfig configures the HTTP server.
|
// HTTPConfig configures the HTTP server.
|
||||||
@@ -58,6 +59,19 @@ type RabbitMQConfig struct {
|
|||||||
Queue string `env:"RABBITMQ_QUEUE" envDefault:"gis.events.example"`
|
Queue string `env:"RABBITMQ_QUEUE" envDefault:"gis.events.example"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LandscapeConfig bounds the Landscape Program's parallelism, and with it the
|
||||||
|
// worker's peak memory: a run holds up to MaxConcurrentNodes nodes in flight,
|
||||||
|
// each running a pool of MaxWorkers/MaxConcurrentNodes goroutines that own one
|
||||||
|
// window buffer per input and output. The defaults (8 total workers spread
|
||||||
|
// over 2 nodes) keep that product bounded regardless of how many cores the
|
||||||
|
// host has.
|
||||||
|
type LandscapeConfig struct {
|
||||||
|
// MaxWorkers is the total window-worker budget shared by all nodes of a run.
|
||||||
|
MaxWorkers int `env:"LP_MAX_WORKERS" envDefault:"8"`
|
||||||
|
// MaxConcurrentNodes is how many graph nodes of one stage run at a time.
|
||||||
|
MaxConcurrentNodes int `env:"LP_MAX_NODES" envDefault:"2"`
|
||||||
|
}
|
||||||
|
|
||||||
// Load reads configuration from the environment, loading an optional .env file
|
// Load reads configuration from the environment, loading an optional .env file
|
||||||
// from the current working directory first.
|
// from the current working directory first.
|
||||||
func Load() (*Config, error) {
|
func Load() (*Config, error) {
|
||||||
|
|||||||
@@ -71,18 +71,30 @@ type DatasetFilter struct {
|
|||||||
FileType *FileType
|
FileType *FileType
|
||||||
Automated *bool
|
Automated *bool
|
||||||
Status *string
|
Status *string
|
||||||
|
// DatasetGroupID restricts the listing to one dataset group.
|
||||||
|
DatasetGroupID *uuid.UUID
|
||||||
|
// StartDateFrom / StartDateTo and EndDateFrom / EndDateTo are inclusive
|
||||||
|
// YYYY-MM-DD bounds on the dataset's period columns.
|
||||||
|
StartDateFrom *string
|
||||||
|
StartDateTo *string
|
||||||
|
EndDateFrom *string
|
||||||
|
EndDateTo *string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Observation is a single unpivoted value from a dataset's attribute table,
|
// Observation is a single unpivoted value from a dataset's attribute table,
|
||||||
// keyed by KATO code and date. Exactly one of Value / ValueText is typically
|
// keyed by KATO code and date (optionally a StartDate..EndDate period). Exactly
|
||||||
// set (numeric vs non-numeric cell); both may be nil for an empty cell.
|
// one of Value / ValueText is typically set (numeric vs non-numeric cell); both
|
||||||
|
// may be nil for an empty cell.
|
||||||
type Observation struct {
|
type Observation struct {
|
||||||
ID uuid.UUID `json:"id"`
|
ID uuid.UUID `json:"id"`
|
||||||
DatasetID uuid.UUID `json:"dataset_id"`
|
DatasetID uuid.UUID `json:"dataset_id"`
|
||||||
KatoCode string `json:"kato_code"`
|
KatoCode string `json:"kato_code"`
|
||||||
Date string `json:"date"`
|
StartDate string `json:"start_date"`
|
||||||
Value *float64 `json:"value"`
|
// EndDate is the period's end date. Nil for a point-in-time observation
|
||||||
ValueText *string `json:"value_text"`
|
// (pre-existing behavior).
|
||||||
|
EndDate *string `json:"end_date,omitempty"`
|
||||||
|
Value *float64 `json:"value"`
|
||||||
|
ValueText *string `json:"value_text"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// allowedExtensions lists the accepted lowercase file extensions (including the
|
// allowedExtensions lists the accepted lowercase file extensions (including the
|
||||||
@@ -115,9 +127,23 @@ func ExtensionAllowedFor(ft FileType, ext string) bool {
|
|||||||
func ValidateFileContent(ext string, head []byte) error {
|
func ValidateFileContent(ext string, head []byte) error {
|
||||||
switch ext {
|
switch ext {
|
||||||
case ".tif", ".tiff":
|
case ".tif", ".tiff":
|
||||||
// TIFF: little-endian "II*\0" or big-endian "MM\0*".
|
// Classic TIFF: little-endian "II*\0" or big-endian "MM\0*".
|
||||||
if !bytes.HasPrefix(head, []byte("II*\x00")) && !bytes.HasPrefix(head, []byte("MM\x00*")) {
|
// BigTIFF (version 43) uses "II+\0" / "MM\0+"; GDAL writes it for
|
||||||
return fmt.Errorf("file is not a valid TIFF/GeoTIFF")
|
// large rasters (BIGTIFF=IF_SAFER), so the pipeline's own COG output
|
||||||
|
// must pass this check too.
|
||||||
|
tiffMagic := [][]byte{
|
||||||
|
[]byte("II*\x00"), []byte("MM\x00*"),
|
||||||
|
[]byte("II+\x00"), []byte("MM\x00+"),
|
||||||
|
}
|
||||||
|
ok := false
|
||||||
|
for _, m := range tiffMagic {
|
||||||
|
if bytes.HasPrefix(head, m) {
|
||||||
|
ok = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("file is not a valid TIFF/BigTIFF GeoTIFF")
|
||||||
}
|
}
|
||||||
case ".zip":
|
case ".zip":
|
||||||
// ZIP local-file or empty-archive signature.
|
// ZIP local-file or empty-archive signature.
|
||||||
@@ -134,10 +160,30 @@ func ValidateFileContent(ext string, head []byte) error {
|
|||||||
if b, ok := firstMeaningfulByte(head); !ok || b != '{' {
|
if b, ok := firstMeaningfulByte(head); !ok || b != '{' {
|
||||||
return fmt.Errorf("file is not valid GeoJSON")
|
return fmt.Errorf("file is not valid GeoJSON")
|
||||||
}
|
}
|
||||||
|
case ".sld", ".xml":
|
||||||
|
// SLD is XML; a lightweight sniff for the opening tag.
|
||||||
|
if b, ok := firstMeaningfulByte(head); !ok || b != '<' {
|
||||||
|
return fmt.Errorf("file is not valid XML/SLD")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AllowedSLDExtensions lists the accepted lowercase extensions (including the
|
||||||
|
// dot) for a dataset's SLD (Styled Layer Descriptor) style file.
|
||||||
|
var AllowedSLDExtensions = []string{".sld", ".xml"}
|
||||||
|
|
||||||
|
// ExtensionAllowedForSLD reports whether ext (lowercase, with dot) is a valid
|
||||||
|
// SLD file extension.
|
||||||
|
func ExtensionAllowedForSLD(ext string) bool {
|
||||||
|
for _, e := range AllowedSLDExtensions {
|
||||||
|
if e == ext {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// firstMeaningfulByte returns the first non-whitespace byte after an optional
|
// firstMeaningfulByte returns the first non-whitespace byte after an optional
|
||||||
// UTF-8 BOM.
|
// UTF-8 BOM.
|
||||||
func firstMeaningfulByte(head []byte) (byte, bool) {
|
func firstMeaningfulByte(head []byte) (byte, bool) {
|
||||||
@@ -160,18 +206,30 @@ type AttributeColumn struct {
|
|||||||
Samples []string `json:"samples,omitempty"`
|
Samples []string `json:"samples,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// YearColumn maps an attribute column to the date it represents,
|
// YearColumn maps an attribute column to the date (or date period) it
|
||||||
// e.g. {"column": "F_2023", "date": "2023-01-01"}.
|
// represents, e.g. {"column": "F_2023", "start_date": "2023-01-01"} for a
|
||||||
|
// single point in time, or {"column": "F_2020_2023", "start_date":
|
||||||
|
// "2020-01-01", "end_date": "2023-12-31"} for a period. EndDate is optional;
|
||||||
|
// when nil the mapping is a single point in time, preserving the original
|
||||||
|
// (pre-period) behavior.
|
||||||
type YearColumn struct {
|
type YearColumn struct {
|
||||||
Column string `json:"column"`
|
Column string `json:"column"`
|
||||||
Date string `json:"date"`
|
StartDate string `json:"start_date"`
|
||||||
|
EndDate *string `json:"end_date,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DatasetSummary is the lightweight view of a dataset used in list responses.
|
// DatasetSummary is the lightweight view of a dataset used in list responses.
|
||||||
// It omits the heavy geometry/attribute/JSONB fields.
|
// It omits the heavy geometry/attribute/JSONB fields.
|
||||||
type DatasetSummary struct {
|
type DatasetSummary struct {
|
||||||
ID uuid.UUID `json:"id"`
|
ID uuid.UUID `json:"id"`
|
||||||
CategoryID uuid.UUID `json:"category_id"`
|
CategoryID uuid.UUID `json:"category_id"`
|
||||||
|
// DatasetGroupID links the dataset to its dataset group ("одноимённые"
|
||||||
|
// datasets versioned by period). Nullable.
|
||||||
|
DatasetGroupID *uuid.UUID `json:"dg_id"`
|
||||||
|
// StartDate / EndDate is the period the dataset describes (YYYY-MM-DD).
|
||||||
|
// Both nullable; EndDate drives the "freshest in group" rule.
|
||||||
|
StartDate *string `json:"start_date"`
|
||||||
|
EndDate *string `json:"end_date"`
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Description *string `json:"description"`
|
Description *string `json:"description"`
|
||||||
Unit *string `json:"unit"`
|
Unit *string `json:"unit"`
|
||||||
@@ -184,21 +242,33 @@ type DatasetSummary struct {
|
|||||||
|
|
||||||
// Dataset is a geo file stored in the object store and grouped under a category.
|
// Dataset is a geo file stored in the object store and grouped under a category.
|
||||||
type Dataset struct {
|
type Dataset struct {
|
||||||
ID uuid.UUID `json:"id"`
|
ID uuid.UUID `json:"id"`
|
||||||
CategoryID uuid.UUID `json:"category_id"`
|
CategoryID uuid.UUID `json:"category_id"`
|
||||||
Name string `json:"name"`
|
// DatasetGroupID links the dataset to its dataset group ("одноимённые"
|
||||||
Description *string `json:"description"`
|
// datasets versioned by period). Nullable.
|
||||||
Unit *string `json:"unit"`
|
DatasetGroupID *uuid.UUID `json:"dg_id"`
|
||||||
Filename string `json:"filename"`
|
// StartDate / EndDate is the period the dataset describes (YYYY-MM-DD).
|
||||||
StorageKey string `json:"storage_key"`
|
// Both nullable; EndDate drives the "freshest in group" rule.
|
||||||
|
StartDate *string `json:"start_date"`
|
||||||
|
EndDate *string `json:"end_date"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Description *string `json:"description"`
|
||||||
|
Unit *string `json:"unit"`
|
||||||
|
Filename string `json:"filename"`
|
||||||
|
StorageKey string `json:"storage_key"`
|
||||||
// CogStorageKey points to the Cloud-Optimized GeoTIFF for rasters. Nullable.
|
// CogStorageKey points to the Cloud-Optimized GeoTIFF for rasters. Nullable.
|
||||||
CogStorageKey *string `json:"cog_storage_key"`
|
CogStorageKey *string `json:"cog_storage_key"`
|
||||||
FileType FileType `json:"file_type"`
|
FileType FileType `json:"file_type"`
|
||||||
SizeBytes int64 `json:"size_bytes"`
|
SizeBytes int64 `json:"size_bytes"`
|
||||||
ContentType string `json:"content_type"`
|
ContentType string `json:"content_type"`
|
||||||
|
// SldStorageKey points to the dataset's SLD (Styled Layer Descriptor) style
|
||||||
|
// file, if one has been uploaded. Nullable; applies to every file type.
|
||||||
|
SldStorageKey *string `json:"sld_storage_key"`
|
||||||
|
SldFilename *string `json:"sld_filename"`
|
||||||
|
SldContentType *string `json:"sld_content_type"`
|
||||||
// Properties holds tabular data extracted from the file (e.g. a shapefile's
|
// Properties holds tabular data extracted from the file (e.g. a shapefile's
|
||||||
// attribute table). Nullable.
|
// attribute table). Nullable; not exposed in dataset responses.
|
||||||
Properties json.RawMessage `json:"properties"`
|
Properties json.RawMessage `json:"-"`
|
||||||
// Meta holds arbitrary user-defined data. Nullable.
|
// Meta holds arbitrary user-defined data. Nullable.
|
||||||
Meta json.RawMessage `json:"meta"`
|
Meta json.RawMessage `json:"meta"`
|
||||||
// Automated is a user-defined flag.
|
// Automated is a user-defined flag.
|
||||||
@@ -215,11 +285,51 @@ type Dataset struct {
|
|||||||
// ParseError holds the failure reason when Status is failed. Nullable.
|
// ParseError holds the failure reason when Status is failed. Nullable.
|
||||||
ParseError *string `json:"parse_error"`
|
ParseError *string `json:"parse_error"`
|
||||||
// Geometry is the dataset's spatial geometry, serialized as GeoJSON.
|
// Geometry is the dataset's spatial geometry, serialized as GeoJSON.
|
||||||
// Nullable; populated from the file's spatial data.
|
// Nullable; populated from the file's spatial data. Not exposed in dataset
|
||||||
Geometry json.RawMessage `json:"geometry"`
|
// responses.
|
||||||
|
Geometry json.RawMessage `json:"-"`
|
||||||
|
// GeoJSON is the pre-assembled GeoJSON FeatureCollection served by the
|
||||||
|
// .geojson endpoint. It is generated and persisted at processing time for
|
||||||
|
// vector and vector_with_kato datasets. Nullable; not exposed on the dataset
|
||||||
|
// response (served only by the .geojson endpoint).
|
||||||
|
GeoJSON json.RawMessage `json:"-"`
|
||||||
// BBox is the axis-aligned bounding box [minX, minY, maxX, maxY] derived
|
// BBox is the axis-aligned bounding box [minX, minY, maxX, maxY] derived
|
||||||
// from the geometry. Included in responses only for raster datasets.
|
// from the geometry. Included in responses only for raster datasets.
|
||||||
BBox []float64 `json:"bbox,omitempty"`
|
BBox []float64 `json:"bbox,omitempty"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
UpdatedAt time.Time `json:"updated_at"`
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DatasetFields is the set of editable metadata columns written by
|
||||||
|
// UpdateFields. Nil pointers clear the corresponding nullable column.
|
||||||
|
type DatasetFields struct {
|
||||||
|
CategoryID uuid.UUID
|
||||||
|
Name string
|
||||||
|
Description *string
|
||||||
|
Unit *string
|
||||||
|
DatasetGroupID *uuid.UUID
|
||||||
|
StartDate *string
|
||||||
|
EndDate *string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateDatePeriod checks that start/end (YYYY-MM-DD, either may be nil)
|
||||||
|
// parse and that end is not before start. Returns an ErrValidation-wrapped
|
||||||
|
// error naming the offending field.
|
||||||
|
func ValidateDatePeriod(start, end *string) error {
|
||||||
|
var from, to time.Time
|
||||||
|
var err error
|
||||||
|
if start != nil {
|
||||||
|
if from, err = time.Parse("2006-01-02", *start); err != nil {
|
||||||
|
return fmt.Errorf("%w: start_date %q is not YYYY-MM-DD", ErrValidation, *start)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if end != nil {
|
||||||
|
if to, err = time.Parse("2006-01-02", *end); err != nil {
|
||||||
|
return fmt.Errorf("%w: end_date %q is not YYYY-MM-DD", ErrValidation, *end)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if start != nil && end != nil && to.Before(from) {
|
||||||
|
return fmt.Errorf("%w: end_date %q is before start_date %q", ErrValidation, *end, *start)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
23
internal/domain/dataset_group.go
Normal file
23
internal/domain/dataset_group.go
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DatasetGroup groups "одноимённые" datasets (the same indicator delivered for
|
||||||
|
// different periods). The freshest dataset of a group is the one with the
|
||||||
|
// greatest end_date. A group may carry a shared SLD style used as a fallback by
|
||||||
|
// datasets that have none of their own.
|
||||||
|
type DatasetGroup struct {
|
||||||
|
ID uuid.UUID `json:"id"`
|
||||||
|
Code string `json:"code"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
SldStorageKey *string `json:"sld_storage_key"`
|
||||||
|
SldFilename *string `json:"sld_filename"`
|
||||||
|
SldContentType *string `json:"sld_content_type"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
}
|
||||||
@@ -1,6 +1,9 @@
|
|||||||
package domain
|
package domain
|
||||||
|
|
||||||
import "testing"
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
func TestFileTypeValid(t *testing.T) {
|
func TestFileTypeValid(t *testing.T) {
|
||||||
valid := []FileType{FileTypeVectorWithKato, FileTypeVector, FileTypeRaster}
|
valid := []FileType{FileTypeVectorWithKato, FileTypeVector, FileTypeRaster}
|
||||||
@@ -27,7 +30,10 @@ func TestValidateFileContent(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"valid tiff LE", ".tif", []byte("II*\x00rest"), true},
|
{"valid tiff LE", ".tif", []byte("II*\x00rest"), true},
|
||||||
{"valid tiff BE", ".tiff", []byte("MM\x00*rest"), true},
|
{"valid tiff BE", ".tiff", []byte("MM\x00*rest"), true},
|
||||||
|
{"valid bigtiff LE", ".tif", []byte("II+\x00rest"), true},
|
||||||
|
{"valid bigtiff BE", ".tiff", []byte("MM\x00+rest"), true},
|
||||||
{"bad tiff", ".tif", []byte("not a tiff"), false},
|
{"bad tiff", ".tif", []byte("not a tiff"), false},
|
||||||
|
{"bad tiff near-miss prefix", ".tif", []byte("II-\x00rest"), false},
|
||||||
{"valid zip", ".zip", []byte("PK\x03\x04rest"), true},
|
{"valid zip", ".zip", []byte("PK\x03\x04rest"), true},
|
||||||
{"bad zip", ".zip", []byte("RARrest"), false},
|
{"bad zip", ".zip", []byte("RARrest"), false},
|
||||||
{"valid gpkg", ".gpkg", []byte("SQLite format 3\x00rest"), true},
|
{"valid gpkg", ".gpkg", []byte("SQLite format 3\x00rest"), true},
|
||||||
@@ -72,3 +78,31 @@ func TestExtensionAllowedFor(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateDatePeriod(t *testing.T) {
|
||||||
|
s := func(v string) *string { return &v }
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
start, end *string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{"both nil", nil, nil, false},
|
||||||
|
{"start only", s("2020-01-01"), nil, false},
|
||||||
|
{"ordered", s("2020-01-01"), s("2020-12-31"), false},
|
||||||
|
{"same day", s("2020-01-01"), s("2020-01-01"), false},
|
||||||
|
{"reversed", s("2020-12-31"), s("2020-01-01"), true},
|
||||||
|
{"bad start", s("01.01.2020"), nil, true},
|
||||||
|
{"bad end", nil, s("2020-13-01"), true},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
err := ValidateDatePeriod(tc.start, tc.end)
|
||||||
|
if (err != nil) != tc.wantErr {
|
||||||
|
t.Fatalf("err=%v wantErr=%v", err, tc.wantErr)
|
||||||
|
}
|
||||||
|
if err != nil && !errors.Is(err, ErrValidation) {
|
||||||
|
t.Fatalf("expected ErrValidation, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
68
internal/domain/landscape.go
Normal file
68
internal/domain/landscape.go
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LPRunStatus is the lifecycle status of a Landscape Program run.
|
||||||
|
type LPRunStatus string
|
||||||
|
|
||||||
|
// Landscape Program run statuses.
|
||||||
|
const (
|
||||||
|
LPRunQueued LPRunStatus = "queued"
|
||||||
|
LPRunRunning LPRunStatus = "running"
|
||||||
|
LPRunSucceeded LPRunStatus = "succeeded"
|
||||||
|
LPRunFailed LPRunStatus = "failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LPNodeStatus is the lifecycle status of a single node within a Landscape
|
||||||
|
// Program run.
|
||||||
|
type LPNodeStatus string
|
||||||
|
|
||||||
|
// Landscape Program run node statuses.
|
||||||
|
const (
|
||||||
|
LPNodePending LPNodeStatus = "pending"
|
||||||
|
LPNodeRunning LPNodeStatus = "running"
|
||||||
|
LPNodeSkipped LPNodeStatus = "skipped"
|
||||||
|
LPNodeSucceeded LPNodeStatus = "succeeded"
|
||||||
|
LPNodeFailed LPNodeStatus = "failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LPDatasetRef references a dataset produced for, or consumed from, a
|
||||||
|
// dataset group slot in the Landscape Program graph.
|
||||||
|
type LPDatasetRef struct {
|
||||||
|
Group string `json:"group"`
|
||||||
|
DatasetID uuid.UUID `json:"dataset_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LPRun is a single execution of the Landscape Program graph.
|
||||||
|
type LPRun struct {
|
||||||
|
ID uuid.UUID `json:"id"`
|
||||||
|
Status LPRunStatus `json:"status"`
|
||||||
|
FromStage *int `json:"from_stage"`
|
||||||
|
Params json.RawMessage `json:"params"`
|
||||||
|
Overrides map[string]uuid.UUID `json:"overrides"`
|
||||||
|
Error *string `json:"error"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
StartedAt *time.Time `json:"started_at"`
|
||||||
|
FinishedAt *time.Time `json:"finished_at"`
|
||||||
|
Nodes []LPRunNode `json:"nodes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LPRunNode is the execution state of a single graph node within an LPRun.
|
||||||
|
type LPRunNode struct {
|
||||||
|
ID uuid.UUID `json:"id"`
|
||||||
|
RunID uuid.UUID `json:"run_id"`
|
||||||
|
NodeCode string `json:"node_code"`
|
||||||
|
Stage int `json:"stage"`
|
||||||
|
Status LPNodeStatus `json:"status"`
|
||||||
|
Fingerprint string `json:"fingerprint"`
|
||||||
|
Inputs []LPDatasetRef `json:"inputs"`
|
||||||
|
Outputs []LPDatasetRef `json:"outputs"`
|
||||||
|
Error *string `json:"error"`
|
||||||
|
StartedAt *time.Time `json:"started_at"`
|
||||||
|
FinishedAt *time.Time `json:"finished_at"`
|
||||||
|
}
|
||||||
225
internal/landscape/compute.go
Normal file
225
internal/landscape/compute.go
Normal file
@@ -0,0 +1,225 @@
|
|||||||
|
package landscape
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"gis/internal/raster"
|
||||||
|
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MaxWorkersPerNode is the per-node worker cap Compute uses. Peak memory is
|
||||||
|
// roughly nodes x workers x window^2 x (inputs+outputs), so callers that run
|
||||||
|
// several nodes at once should divide their total worker budget between them
|
||||||
|
// and call ComputeWithWorkers instead.
|
||||||
|
const MaxWorkersPerNode = 8
|
||||||
|
|
||||||
|
// Compute runs one node over its input rasters with the default per-node
|
||||||
|
// worker cap (MaxWorkersPerNode). See ComputeWithWorkers.
|
||||||
|
func Compute(ctx context.Context, rio raster.IO, node Node, inputs map[string]string, outDir string, p Params) (map[string]string, error) {
|
||||||
|
return ComputeWithWorkers(ctx, rio, node, inputs, outDir, p, MaxWorkersPerNode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComputeWithWorkers runs one node over its input rasters with an explicit
|
||||||
|
// worker cap. See the package doc for the concurrency model: windows are
|
||||||
|
// processed by a worker pool of at most workers goroutines; each worker owns
|
||||||
|
// its own Reader handles (GDAL datasets are not goroutine-safe) and writes are
|
||||||
|
// serialised per output. Window order does not affect the result. On any
|
||||||
|
// error, no output files are left behind: ComputeWithWorkers best-effort
|
||||||
|
// removes every output it created before returning. A workers value below 1 is
|
||||||
|
// treated as 1.
|
||||||
|
func ComputeWithWorkers(ctx context.Context, rio raster.IO, node Node, inputs map[string]string, outDir string, p Params, workers int) (map[string]string, error) {
|
||||||
|
p = p.WithDefaults()
|
||||||
|
if p.WindowSize <= 0 {
|
||||||
|
// WithDefaults only fills in a zero value; guard against a negative
|
||||||
|
// one too, since raster.Windows would otherwise loop forever.
|
||||||
|
p.WindowSize = DefaultParams().WindowSize
|
||||||
|
}
|
||||||
|
paths := make([]string, len(node.Inputs))
|
||||||
|
for i, g := range node.Inputs {
|
||||||
|
path, ok := inputs[g]
|
||||||
|
if !ok || path == "" {
|
||||||
|
return nil, fmt.Errorf("node %s (%s): missing input %q", node.Code, node.Name, g)
|
||||||
|
}
|
||||||
|
paths[i] = path
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate grids with one set of handles before creating anything.
|
||||||
|
grid, err := checkGrids(ctx, rio, node, paths)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
outPaths := make(map[string]string, len(node.Outputs))
|
||||||
|
writers := make([]raster.Writer, len(node.Outputs))
|
||||||
|
createdPaths := make([]string, 0, len(node.Outputs))
|
||||||
|
for j, g := range node.Outputs {
|
||||||
|
path := filepath.Join(outDir, g+".tif")
|
||||||
|
w, err := rio.Create(ctx, path, grid, raster.DefaultWriterOptions())
|
||||||
|
if err != nil {
|
||||||
|
closeWriters(writers[:j])
|
||||||
|
removeOutputs(createdPaths)
|
||||||
|
return nil, fmt.Errorf("node %s: create output %s: %w", node.Code, g, err)
|
||||||
|
}
|
||||||
|
writers[j] = w
|
||||||
|
outPaths[g] = path
|
||||||
|
createdPaths = append(createdPaths, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
windows := raster.Windows(grid.Width, grid.Height, p.WindowSize)
|
||||||
|
// The first window raster.Windows produces is always the largest: it
|
||||||
|
// starts at (0,0) so both its width and height are the un-clipped
|
||||||
|
// min(WindowSize, dimension). Size per-worker buffers off it instead of
|
||||||
|
// WindowSize*WindowSize, which would over-allocate whenever the raster is
|
||||||
|
// smaller than one window (the common case for small/test rasters, and
|
||||||
|
// for the last row/column of any raster).
|
||||||
|
var size int
|
||||||
|
if len(windows) > 0 {
|
||||||
|
size = windows[0].Width * windows[0].Height
|
||||||
|
}
|
||||||
|
jobs := make(chan raster.Window)
|
||||||
|
var writeMu sync.Mutex
|
||||||
|
eg, gctx := errgroup.WithContext(ctx)
|
||||||
|
// Cap workers well below GOMAXPROCS too: each worker holds a full set of
|
||||||
|
// per-window input/output buffers, and nodes can have many inputs and
|
||||||
|
// outputs (e.g. node 20: 6 inputs, 5 outputs), so an unbounded worker
|
||||||
|
// count multiplies memory with core count and with concurrently running
|
||||||
|
// nodes. The caller's budget is the third bound.
|
||||||
|
workers = min(runtime.GOMAXPROCS(0), len(windows), max(1, workers))
|
||||||
|
for w := 0; w < workers; w++ {
|
||||||
|
eg.Go(func() error {
|
||||||
|
readers, err := openAll(gctx, rio, node, paths)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer closeReaders(readers)
|
||||||
|
in := make([][]float64, len(paths))
|
||||||
|
for i := range in {
|
||||||
|
in[i] = make([]float64, size)
|
||||||
|
}
|
||||||
|
out := make([][]int16, len(writers))
|
||||||
|
for j := range out {
|
||||||
|
out[j] = make([]int16, size)
|
||||||
|
}
|
||||||
|
for win := range jobs {
|
||||||
|
n := win.Width * win.Height
|
||||||
|
inView := make([][]float64, len(in))
|
||||||
|
for i, r := range readers {
|
||||||
|
if err := r.ReadFloat64(win, in[i][:n]); err != nil {
|
||||||
|
return fmt.Errorf("node %s: read %s window %+v: %w", node.Code, node.Inputs[i], win, err)
|
||||||
|
}
|
||||||
|
inView[i] = in[i][:n]
|
||||||
|
}
|
||||||
|
outView := make([][]int16, len(out))
|
||||||
|
for j := range out {
|
||||||
|
outView[j] = out[j][:n]
|
||||||
|
}
|
||||||
|
node.Kernel(inView, outView, p)
|
||||||
|
writeMu.Lock()
|
||||||
|
for j, wr := range writers {
|
||||||
|
if err := wr.WriteInt16(win, outView[j]); err != nil {
|
||||||
|
writeMu.Unlock()
|
||||||
|
return fmt.Errorf("node %s: write %s window %+v: %w", node.Code, node.Outputs[j], win, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeMu.Unlock()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
eg.Go(func() error {
|
||||||
|
defer close(jobs)
|
||||||
|
for _, win := range windows {
|
||||||
|
select {
|
||||||
|
case jobs <- win:
|
||||||
|
case <-gctx.Done():
|
||||||
|
return gctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
runErr := eg.Wait()
|
||||||
|
if err := closeWriters(writers); err != nil && runErr == nil {
|
||||||
|
runErr = err
|
||||||
|
}
|
||||||
|
if runErr != nil {
|
||||||
|
removeOutputs(createdPaths)
|
||||||
|
return nil, runErr
|
||||||
|
}
|
||||||
|
return outPaths, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkGrids opens every input once, verifies all grids match the first, and
|
||||||
|
// returns that grid.
|
||||||
|
func checkGrids(ctx context.Context, rio raster.IO, node Node, paths []string) (raster.Grid, error) {
|
||||||
|
readers, err := openAll(ctx, rio, node, paths)
|
||||||
|
if err != nil {
|
||||||
|
return raster.Grid{}, err
|
||||||
|
}
|
||||||
|
defer closeReaders(readers)
|
||||||
|
if len(readers) == 0 {
|
||||||
|
return raster.Grid{}, fmt.Errorf("node %s (%s): has no inputs", node.Code, node.Name)
|
||||||
|
}
|
||||||
|
ref := readers[0].Grid()
|
||||||
|
for i := 1; i < len(readers); i++ {
|
||||||
|
if g := readers[i].Grid(); !rio.SameGrid(ref, g) {
|
||||||
|
return raster.Grid{}, fmt.Errorf("node %s: input %s grid (%dx%d, %v) differs from %s (%dx%d, %v); inputs must be aligned",
|
||||||
|
node.Code, node.Inputs[i], g.Width, g.Height, g.GeoTransform, node.Inputs[0], ref.Width, ref.Height, ref.GeoTransform)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ref, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// openAll opens paths in order, wrapping any failure with the node code and
|
||||||
|
// the input group name (paths is always node.Inputs resolved to files, in
|
||||||
|
// the same order) for context.
|
||||||
|
func openAll(ctx context.Context, rio raster.IO, node Node, paths []string) ([]raster.Reader, error) {
|
||||||
|
readers := make([]raster.Reader, 0, len(paths))
|
||||||
|
for i, p := range paths {
|
||||||
|
r, err := rio.Open(ctx, p)
|
||||||
|
if err != nil {
|
||||||
|
closeReaders(readers)
|
||||||
|
name := p
|
||||||
|
if i < len(node.Inputs) {
|
||||||
|
name = node.Inputs[i]
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("node %s: input %s: %w", node.Code, name, err)
|
||||||
|
}
|
||||||
|
readers = append(readers, r)
|
||||||
|
}
|
||||||
|
return readers, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func closeReaders(rs []raster.Reader) {
|
||||||
|
for _, r := range rs {
|
||||||
|
if r != nil {
|
||||||
|
_ = r.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func closeWriters(ws []raster.Writer) error {
|
||||||
|
var first error
|
||||||
|
for _, w := range ws {
|
||||||
|
if w == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := w.Close(); err != nil && first == nil {
|
||||||
|
first = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeOutputs best-effort deletes every path in paths, ignoring errors. It
|
||||||
|
// is used to leave no partially-written output behind after a failed run.
|
||||||
|
func removeOutputs(paths []string) {
|
||||||
|
for _, p := range paths {
|
||||||
|
_ = os.Remove(p)
|
||||||
|
}
|
||||||
|
}
|
||||||
252
internal/landscape/compute_test.go
Normal file
252
internal/landscape/compute_test.go
Normal file
@@ -0,0 +1,252 @@
|
|||||||
|
package landscape
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gis/internal/raster"
|
||||||
|
)
|
||||||
|
|
||||||
|
func grid(w, h int) raster.Grid {
|
||||||
|
return raster.Grid{Width: w, Height: h, GeoTransform: [6]float64{0, 30, 0, 0, 0, -30}, ProjectionWKT: "EPSG:32642"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func constRaster(w, h int, v float64) []float64 {
|
||||||
|
d := make([]float64, w*h)
|
||||||
|
for i := range d {
|
||||||
|
d[i] = v
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompute_E_TilesWindowsAndWritesEveryPixel(t *testing.T) {
|
||||||
|
fio := newFakeIO()
|
||||||
|
g := grid(10, 7)
|
||||||
|
z := constRaster(10, 7, 2)
|
||||||
|
u := constRaster(10, 7, 4)
|
||||||
|
z[0] = -9999 // pixel 0 invalid
|
||||||
|
fio.put("z.tif", g, z)
|
||||||
|
fio.put("u.tif", g, u)
|
||||||
|
node, _ := NodeByCode("10")
|
||||||
|
p := DefaultParams()
|
||||||
|
p.WindowSize = 4 // forces partial windows (10x7 with 4x4 tiles)
|
||||||
|
out, err := Compute(context.Background(), fio, node, map[string]string{OutZIntegrated: "z.tif", OutUIntegrated: "u.tif"}, "/out", p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path := out[OutE]
|
||||||
|
if path != filepath.Join("/out", OutE+".tif") {
|
||||||
|
t.Fatalf("unexpected output path %q", path)
|
||||||
|
}
|
||||||
|
res := fio.files[path]
|
||||||
|
if res.grid != g {
|
||||||
|
t.Fatal("output grid must copy the first input's grid")
|
||||||
|
}
|
||||||
|
if res.opts != raster.DefaultWriterOptions() {
|
||||||
|
t.Fatalf("output created with %+v, want raster.DefaultWriterOptions() = %+v", res.opts, raster.DefaultWriterOptions())
|
||||||
|
}
|
||||||
|
for i, v := range res.data {
|
||||||
|
want := 4.0
|
||||||
|
if i == 0 {
|
||||||
|
want = -9999
|
||||||
|
}
|
||||||
|
if v != want {
|
||||||
|
t.Fatalf("pixel %d = %v want %v", i, v, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompute_FailsOnGridMismatch(t *testing.T) {
|
||||||
|
fio := newFakeIO()
|
||||||
|
fio.put("z.tif", grid(10, 7), constRaster(10, 7, 2))
|
||||||
|
fio.put("u.tif", grid(9, 7), constRaster(9, 7, 4))
|
||||||
|
node, _ := NodeByCode("10")
|
||||||
|
_, err := Compute(context.Background(), fio, node, map[string]string{OutZIntegrated: "z.tif", OutUIntegrated: "u.tif"}, "/out", DefaultParams())
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected grid mismatch error")
|
||||||
|
}
|
||||||
|
if _, created := fio.files[filepath.Join("/out", OutE+".tif")]; created {
|
||||||
|
t.Fatal("no output must be created on mismatch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompute_FailsOnMissingInput(t *testing.T) {
|
||||||
|
fio := newFakeIO()
|
||||||
|
fio.put("z.tif", grid(2, 2), constRaster(2, 2, 2))
|
||||||
|
node, _ := NodeByCode("10")
|
||||||
|
if _, err := Compute(context.Background(), fio, node, map[string]string{OutZIntegrated: "z.tif"}, "/out", DefaultParams()); err == nil {
|
||||||
|
t.Fatal("expected error for missing input group")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompute_MultiOutputNode(t *testing.T) {
|
||||||
|
fio := newFakeIO()
|
||||||
|
g := grid(3, 1)
|
||||||
|
paths := map[string]string{}
|
||||||
|
for i, s := range []string{SlotPIrrigation, SlotPSnow, SlotPCrop, SlotPPasture, SlotPTourism} {
|
||||||
|
paths[s] = s + ".tif"
|
||||||
|
fio.put(paths[s], g, constRaster(3, 1, float64(i+1)))
|
||||||
|
}
|
||||||
|
paths[OutE] = "e.tif"
|
||||||
|
fio.put("e.tif", g, constRaster(3, 1, 3))
|
||||||
|
node, _ := NodeByCode("20")
|
||||||
|
out, err := Compute(context.Background(), fio, node, paths, "/o", DefaultParams())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(out) != 5 {
|
||||||
|
t.Fatalf("want 5 outputs, got %d", len(out))
|
||||||
|
}
|
||||||
|
if fio.files[out[OutCTourism]].data[0] != 3 || fio.files[out[OutCIrrigation]].data[0] != 1 {
|
||||||
|
t.Fatal("min(P, E) not applied per output")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCompute_MultiOutputNode_TiledWindows runs the 6-input/5-output node 20
|
||||||
|
// with a window size much smaller than the raster, so per-worker output
|
||||||
|
// buffers are reused across windows of differing pixel counts (partial
|
||||||
|
// windows at the raster's right/bottom edges).
|
||||||
|
func TestCompute_MultiOutputNode_TiledWindows(t *testing.T) {
|
||||||
|
fio := newFakeIO()
|
||||||
|
w, h := 5, 3
|
||||||
|
g := grid(w, h)
|
||||||
|
paths := map[string]string{}
|
||||||
|
for i, s := range []string{SlotPIrrigation, SlotPSnow, SlotPCrop, SlotPPasture, SlotPTourism} {
|
||||||
|
paths[s] = s + ".tif"
|
||||||
|
fio.put(paths[s], g, constRaster(w, h, float64(i+1)))
|
||||||
|
}
|
||||||
|
paths[OutE] = "e.tif"
|
||||||
|
fio.put("e.tif", g, constRaster(w, h, 3))
|
||||||
|
node, _ := NodeByCode("20")
|
||||||
|
p := DefaultParams()
|
||||||
|
p.WindowSize = 2 // 5x3 with 2x2 tiles: several differently-sized windows
|
||||||
|
out, err := Compute(context.Background(), fio, node, paths, "/o2", p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
codes := []string{OutCIrrigation, OutCSnow, OutCCrop, OutCPasture, OutCTourism}
|
||||||
|
for i, code := range codes {
|
||||||
|
want := min(i+1, 3) // min(P_sector, E=3)
|
||||||
|
res := fio.files[out[code]]
|
||||||
|
for px, v := range res.data {
|
||||||
|
if v != float64(want) {
|
||||||
|
t.Fatalf("%s pixel %d = %v want %v", code, px, v, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCompute_ZeroParamsUsesDefaults exercises Compute with a completely
|
||||||
|
// zero-valued Params, relying on WithDefaults to fill in WindowSize and the
|
||||||
|
// other node-independent knobs.
|
||||||
|
func TestCompute_ZeroParamsUsesDefaults(t *testing.T) {
|
||||||
|
fio := newFakeIO()
|
||||||
|
g := grid(4, 4)
|
||||||
|
fio.put("z.tif", g, constRaster(4, 4, 2))
|
||||||
|
fio.put("u.tif", g, constRaster(4, 4, 3))
|
||||||
|
node, _ := NodeByCode("10")
|
||||||
|
out, err := Compute(context.Background(), fio, node, map[string]string{OutZIntegrated: "z.tif", OutUIntegrated: "u.tif"}, "/z", Params{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, v := range fio.files[out[OutE]].data {
|
||||||
|
if v != 3 {
|
||||||
|
t.Fatalf("pixel = %v want 3", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCompute_NegativeWindowSizeClampedToDefault guards against
|
||||||
|
// raster.Windows looping forever on a non-positive window size: WithDefaults
|
||||||
|
// only fills in a zero value, so Compute must clamp negatives itself.
|
||||||
|
func TestCompute_NegativeWindowSizeClampedToDefault(t *testing.T) {
|
||||||
|
fio := newFakeIO()
|
||||||
|
g := grid(2, 2)
|
||||||
|
fio.put("z.tif", g, constRaster(2, 2, 2))
|
||||||
|
fio.put("u.tif", g, constRaster(2, 2, 3))
|
||||||
|
node, _ := NodeByCode("10")
|
||||||
|
p := DefaultParams()
|
||||||
|
p.WindowSize = -5
|
||||||
|
out, err := Compute(context.Background(), fio, node, map[string]string{OutZIntegrated: "z.tif", OutUIntegrated: "u.tif"}, "/neg", p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if fio.files[out[OutE]].data[0] != 3 {
|
||||||
|
t.Fatal("unexpected result")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCompute_FailureMidRunReturnsError simulates a read failure partway
|
||||||
|
// through a run (the fake fails every ReadFloat64 against u.tif from the
|
||||||
|
// first call on) and checks Compute surfaces the error instead of returning
|
||||||
|
// a partial result.
|
||||||
|
func TestCompute_FailureMidRunReturnsError(t *testing.T) {
|
||||||
|
fio := newFakeIO()
|
||||||
|
g := grid(10, 7)
|
||||||
|
fio.put("z.tif", g, constRaster(10, 7, 2))
|
||||||
|
fio.put("u.tif", g, constRaster(10, 7, 4))
|
||||||
|
fio.failReadAfterN("u.tif", 0) // fail starting from the very first read
|
||||||
|
node, _ := NodeByCode("10")
|
||||||
|
p := DefaultParams()
|
||||||
|
p.WindowSize = 4
|
||||||
|
if _, err := Compute(context.Background(), fio, node, map[string]string{OutZIntegrated: "z.tif", OutUIntegrated: "u.tif"}, "/out-fail", p); err == nil {
|
||||||
|
t.Fatal("expected error from simulated read failure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRemoveOutputs_DeletesFiles unit-tests the cleanup helper Compute uses
|
||||||
|
// after a failed run, against the real filesystem (the fake IO above has no
|
||||||
|
// on-disk files to assert against).
|
||||||
|
func TestRemoveOutputs_DeletesFiles(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
var paths []string
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
p := filepath.Join(dir, fmt.Sprintf("out%d.tif", i))
|
||||||
|
if err := os.WriteFile(p, []byte("data"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
paths = append(paths, p)
|
||||||
|
}
|
||||||
|
removeOutputs(paths)
|
||||||
|
for _, p := range paths {
|
||||||
|
if _, err := os.Stat(p); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected %s to be removed, stat err = %v", p, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestComputeWithWorkers_RespectsCap checks the caller's worker budget bounds
|
||||||
|
// the pool: each pool worker opens one reader per input and holds a buffer per
|
||||||
|
// input and output, so the open count bounds peak memory. checkGrids opens one
|
||||||
|
// extra set of handles before the pool starts, hence the (workers+1) bound.
|
||||||
|
func TestComputeWithWorkers_RespectsCap(t *testing.T) {
|
||||||
|
node, _ := NodeByCode("10")
|
||||||
|
inputs := len(node.Inputs)
|
||||||
|
|
||||||
|
for _, workers := range []int{1, 2, 4, 0, -3} {
|
||||||
|
t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) {
|
||||||
|
fio := newFakeIO()
|
||||||
|
g := grid(64, 64)
|
||||||
|
fio.put("z.tif", g, constRaster(64, 64, 2))
|
||||||
|
fio.put("u.tif", g, constRaster(64, 64, 4))
|
||||||
|
p := DefaultParams()
|
||||||
|
p.WindowSize = 8 // 64 windows, so the pool is never window-bound
|
||||||
|
|
||||||
|
if _, err := ComputeWithWorkers(context.Background(), fio, node,
|
||||||
|
map[string]string{OutZIntegrated: "z.tif", OutUIntegrated: "u.tif"}, "/out", p, workers); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
effective := max(1, workers)
|
||||||
|
if got, limit := fio.openCount(), (effective+1)*inputs; got > limit {
|
||||||
|
t.Fatalf("opened %d readers with workers=%d, want at most %d", got, workers, limit)
|
||||||
|
}
|
||||||
|
if fio.openCount() < inputs {
|
||||||
|
t.Fatalf("opened %d readers, want at least one set of %d", fio.openCount(), inputs)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
122
internal/landscape/fakeio_test.go
Normal file
122
internal/landscape/fakeio_test.go
Normal file
@@ -0,0 +1,122 @@
|
|||||||
|
package landscape
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"gis/internal/raster"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeIO keeps rasters in memory keyed by path.
|
||||||
|
type fakeIO struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
files map[string]*fakeRaster
|
||||||
|
|
||||||
|
// failReadAfter, when set for a path, makes every ReadFloat64 call
|
||||||
|
// against that path fail once the cumulative read count (across all
|
||||||
|
// readers/workers) exceeds the configured threshold. Used to simulate a
|
||||||
|
// mid-run I/O failure.
|
||||||
|
failReadAfter map[string]int
|
||||||
|
readCounts map[string]int
|
||||||
|
|
||||||
|
// opens counts every successful Open. Each pool worker opens one reader
|
||||||
|
// per input, so the count bounds the number of concurrent handles.
|
||||||
|
opens int
|
||||||
|
}
|
||||||
|
|
||||||
|
// openCount returns how many readers have been opened so far.
|
||||||
|
func (f *fakeIO) openCount() int {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return f.opens
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeRaster struct {
|
||||||
|
grid raster.Grid
|
||||||
|
data []float64 // row-major
|
||||||
|
opts raster.WriterOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFakeIO() *fakeIO { return &fakeIO{files: map[string]*fakeRaster{}} }
|
||||||
|
|
||||||
|
func (f *fakeIO) put(path string, g raster.Grid, data []float64) {
|
||||||
|
f.files[path] = &fakeRaster{grid: g, data: data}
|
||||||
|
}
|
||||||
|
|
||||||
|
// failReadAfterN makes ReadFloat64 against path fail starting from the
|
||||||
|
// (n+1)-th call to it.
|
||||||
|
func (f *fakeIO) failReadAfterN(path string, n int) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
if f.failReadAfter == nil {
|
||||||
|
f.failReadAfter = map[string]int{}
|
||||||
|
f.readCounts = map[string]int{}
|
||||||
|
}
|
||||||
|
f.failReadAfter[path] = n
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeReader struct {
|
||||||
|
f *fakeIO
|
||||||
|
path string
|
||||||
|
r *fakeRaster
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *fakeReader) Grid() raster.Grid { return r.r.grid }
|
||||||
|
func (r *fakeReader) NoData() (float64, bool) { return -9999, true }
|
||||||
|
func (r *fakeReader) Close() error { return nil }
|
||||||
|
func (r *fakeReader) ReadFloat64(w raster.Window, dst []float64) error {
|
||||||
|
if r.f != nil {
|
||||||
|
r.f.mu.Lock()
|
||||||
|
if n, ok := r.f.failReadAfter[r.path]; ok {
|
||||||
|
r.f.readCounts[r.path]++
|
||||||
|
if r.f.readCounts[r.path] > n {
|
||||||
|
r.f.mu.Unlock()
|
||||||
|
return fmt.Errorf("fake: simulated read failure for %s", r.path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r.f.mu.Unlock()
|
||||||
|
}
|
||||||
|
for y := 0; y < w.Height; y++ {
|
||||||
|
copy(dst[y*w.Width:(y+1)*w.Width], r.r.data[(w.Y+y)*r.r.grid.Width+w.X:(w.Y+y)*r.r.grid.Width+w.X+w.Width])
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeWriter struct {
|
||||||
|
f *fakeIO
|
||||||
|
r *fakeRaster
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *fakeWriter) WriteInt16(win raster.Window, src []int16) error {
|
||||||
|
w.f.mu.Lock()
|
||||||
|
defer w.f.mu.Unlock()
|
||||||
|
for y := 0; y < win.Height; y++ {
|
||||||
|
for x := 0; x < win.Width; x++ {
|
||||||
|
w.r.data[(win.Y+y)*w.r.grid.Width+win.X+x] = float64(src[y*win.Width+x])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (w *fakeWriter) Close() error { return nil }
|
||||||
|
|
||||||
|
func (f *fakeIO) Open(_ context.Context, path string) (raster.Reader, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
r, ok := f.files[path]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("fake: %s not found", path)
|
||||||
|
}
|
||||||
|
f.opens++
|
||||||
|
return &fakeReader{f: f, path: path, r: r}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeIO) Create(_ context.Context, path string, g raster.Grid, o raster.WriterOptions) (raster.Writer, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
r := &fakeRaster{grid: g, data: make([]float64, g.Width*g.Height), opts: o}
|
||||||
|
f.files[path] = r
|
||||||
|
return &fakeWriter{f: f, r: r}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeIO) SameGrid(a, b raster.Grid) bool { return a == b }
|
||||||
157
internal/landscape/graph.go
Normal file
157
internal/landscape/graph.go
Normal file
@@ -0,0 +1,157 @@
|
|||||||
|
// Package landscape defines the Landscape Program (ЛП) computation graph — the
|
||||||
|
// per-pixel formulas that turn ecological (Z), vulnerability (U) and
|
||||||
|
// suitability (P) class rasters into constraints (E), conflicts (C) and
|
||||||
|
// functional zoning (V1–V3) — together with a window-based executor.
|
||||||
|
package landscape
|
||||||
|
|
||||||
|
import "sort"
|
||||||
|
|
||||||
|
const (
|
||||||
|
// NoData is the output nodata value of every LP raster.
|
||||||
|
NoData int16 = -9999
|
||||||
|
// CategoryCode is the dataset category holding every LP dataset.
|
||||||
|
CategoryCode = "landscape"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Group codes. Slots are leaf inputs users upload; the rest are node outputs.
|
||||||
|
const (
|
||||||
|
SlotZBiodiversity = "lp-z-biodiversity"
|
||||||
|
SlotZLCR = "lp-z-lcr"
|
||||||
|
SlotZSoilProtection = "lp-z-soil-protection"
|
||||||
|
SlotZWR = "lp-z-wr"
|
||||||
|
SlotUSurfWater = "lp-u-surf-water"
|
||||||
|
SlotUSoil = "lp-u-soil"
|
||||||
|
SlotUCrop = "lp-u-crop"
|
||||||
|
SlotUVeg = "lp-u-veg"
|
||||||
|
SlotPIrrigation = "lp-p-irrigation"
|
||||||
|
SlotPSnow = "lp-p-snow"
|
||||||
|
SlotPCrop = "lp-p-crop"
|
||||||
|
SlotPPasture = "lp-p-pasture"
|
||||||
|
SlotPTourism = "lp-p-tourism"
|
||||||
|
|
||||||
|
OutZIntegrated = "lp-z-integrated"
|
||||||
|
OutUIntegrated = "lp-u-integrated"
|
||||||
|
OutZMax = "lp-z-max"
|
||||||
|
OutUMax = "lp-u-max"
|
||||||
|
OutE = "lp-e"
|
||||||
|
OutCIrrigation = "lp-c-irrigation"
|
||||||
|
OutCSnow = "lp-c-snow"
|
||||||
|
OutCCrop = "lp-c-crop"
|
||||||
|
OutCPasture = "lp-c-pasture"
|
||||||
|
OutCTourism = "lp-c-tourism"
|
||||||
|
OutPMax = "lp-p-max"
|
||||||
|
OutV1 = "lp-v1-conservative"
|
||||||
|
OutV2 = "lp-v2-balanced"
|
||||||
|
OutV3 = "lp-v3-development"
|
||||||
|
OutCMax = "lp-c-max"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Slot struct{ Code, Title string }
|
||||||
|
type Output struct{ Code, Title string }
|
||||||
|
|
||||||
|
// Kernel computes one window. in[i] follows Node.Inputs order and out[j]
|
||||||
|
// follows Node.Outputs order; every slice has exactly window-pixel length.
|
||||||
|
type Kernel func(in [][]float64, out [][]int16, p Params)
|
||||||
|
|
||||||
|
// Node is one function of the graph. Stage = first digit of Code.
|
||||||
|
type Node struct {
|
||||||
|
Code, Name string
|
||||||
|
Stage int
|
||||||
|
Inputs []string
|
||||||
|
Outputs []string
|
||||||
|
Kernel Kernel
|
||||||
|
}
|
||||||
|
|
||||||
|
var Slots = []Slot{
|
||||||
|
{SlotZBiodiversity, "Z Biodiversity"}, {SlotZLCR, "Z LCR"}, {SlotZSoilProtection, "Z Soil Protection"}, {SlotZWR, "Z Water Resources"},
|
||||||
|
{SlotUSurfWater, "U Surface Water"}, {SlotUSoil, "U Soil"}, {SlotUCrop, "U Crop"}, {SlotUVeg, "U Vegetation"},
|
||||||
|
{SlotPIrrigation, "P Irrigation"}, {SlotPSnow, "P Snow"}, {SlotPCrop, "P Crop"}, {SlotPPasture, "P Pasture"}, {SlotPTourism, "P Tourism"},
|
||||||
|
}
|
||||||
|
|
||||||
|
var Outputs = []Output{
|
||||||
|
{OutZIntegrated, "Z Integrated"}, {OutUIntegrated, "U Integrated"}, {OutZMax, "Z Max"}, {OutUMax, "U Max"},
|
||||||
|
{OutE, "E Ecological Constraint"},
|
||||||
|
{OutCIrrigation, "C Irrigation"}, {OutCSnow, "C Snow"}, {OutCCrop, "C Crop"}, {OutCPasture, "C Pasture"}, {OutCTourism, "C Tourism"},
|
||||||
|
{OutPMax, "P Max"},
|
||||||
|
{OutV1, "V1 Conservative"}, {OutV2, "V2 Balanced"}, {OutV3, "V3 Development"}, {OutCMax, "C Max"},
|
||||||
|
}
|
||||||
|
|
||||||
|
var zInputs = []string{SlotZBiodiversity, SlotZLCR, SlotZSoilProtection, SlotZWR}
|
||||||
|
var uInputs = []string{SlotUSurfWater, SlotUSoil, SlotUCrop, SlotUVeg} // order matters: water, soil, crop, veg
|
||||||
|
var pInputs = []string{SlotPIrrigation, SlotPSnow, SlotPCrop, SlotPPasture, SlotPTourism} // sector order 1..5
|
||||||
|
var cOutputs = []string{OutCIrrigation, OutCSnow, OutCCrop, OutCPasture, OutCTourism}
|
||||||
|
|
||||||
|
var Nodes = []Node{
|
||||||
|
{Code: "00", Name: "z_integrated", Stage: 0, Inputs: zInputs, Outputs: []string{OutZIntegrated}, Kernel: kernelZIntegrated},
|
||||||
|
{Code: "01", Name: "u_integrated", Stage: 0, Inputs: uInputs, Outputs: []string{OutUIntegrated}, Kernel: kernelUIntegrated},
|
||||||
|
{Code: "02", Name: "z_max", Stage: 0, Inputs: zInputs, Outputs: []string{OutZMax}, Kernel: kernelZMax},
|
||||||
|
{Code: "03", Name: "u_max", Stage: 0, Inputs: uInputs, Outputs: []string{OutUMax}, Kernel: kernelUMax},
|
||||||
|
{Code: "10", Name: "e_ecological_constraint", Stage: 1, Inputs: []string{OutZIntegrated, OutUIntegrated}, Outputs: []string{OutE}, Kernel: kernelE},
|
||||||
|
{Code: "20", Name: "c_conflicts", Stage: 2, Inputs: append(append([]string{}, pInputs...), OutE), Outputs: cOutputs, Kernel: kernelCConflicts},
|
||||||
|
{Code: "21", Name: "p_max", Stage: 2, Inputs: pInputs, Outputs: []string{OutPMax}, Kernel: kernelMaxAnyValid},
|
||||||
|
// func_zoning ranks the original suitability rasters P1..P5 against E (the
|
||||||
|
// reference notebook and zoning_core.py); the conflict layers C_k are
|
||||||
|
// diagnostics and are not zoning inputs. It stays in stage 3 to keep the
|
||||||
|
// diagram's folder numbering (30x) even though it only depends on stage 1.
|
||||||
|
{Code: "30", Name: "func_zoning", Stage: 3, Inputs: append([]string{OutE}, pInputs...), Outputs: []string{OutV1, OutV2, OutV3}, Kernel: kernelFuncZoning},
|
||||||
|
{Code: "31", Name: "c_max", Stage: 3, Inputs: cOutputs, Outputs: []string{OutCMax}, Kernel: kernelMaxAnyValid},
|
||||||
|
}
|
||||||
|
|
||||||
|
func NodeByCode(code string) (Node, bool) {
|
||||||
|
for _, n := range Nodes {
|
||||||
|
if n.Code == code {
|
||||||
|
return n, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Node{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func SlotByCode(code string) (Slot, bool) {
|
||||||
|
for _, s := range Slots {
|
||||||
|
if s.Code == code {
|
||||||
|
return s, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Slot{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func OutputByCode(code string) (Output, bool) {
|
||||||
|
for _, o := range Outputs {
|
||||||
|
if o.Code == code {
|
||||||
|
return o, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Output{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// NodeForOutput returns the node producing the given output group.
|
||||||
|
func NodeForOutput(group string) (Node, bool) {
|
||||||
|
for _, n := range Nodes {
|
||||||
|
for _, o := range n.Outputs {
|
||||||
|
if o == group {
|
||||||
|
return n, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Node{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stages groups nodes by stage, ascending; within a stage nodes are sorted by code.
|
||||||
|
func Stages() [][]Node {
|
||||||
|
byStage := map[int][]Node{}
|
||||||
|
for _, n := range Nodes {
|
||||||
|
byStage[n.Stage] = append(byStage[n.Stage], n)
|
||||||
|
}
|
||||||
|
keys := make([]int, 0, len(byStage))
|
||||||
|
for k := range byStage {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Ints(keys)
|
||||||
|
out := make([][]Node, 0, len(keys))
|
||||||
|
for _, k := range keys {
|
||||||
|
ns := byStage[k]
|
||||||
|
sort.Slice(ns, func(i, j int) bool { return ns[i].Code < ns[j].Code })
|
||||||
|
out = append(out, ns)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
153
internal/landscape/graph_test.go
Normal file
153
internal/landscape/graph_test.go
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
package landscape
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
var slugRe = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`)
|
||||||
|
|
||||||
|
// isSlot reports whether code names one of the leaf input slots.
|
||||||
|
func isSlot(code string) bool {
|
||||||
|
_, ok := SlotByCode(code)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNodeInputsComeFromSlotsOrLowerStage(t *testing.T) {
|
||||||
|
for _, n := range Nodes {
|
||||||
|
for _, in := range n.Inputs {
|
||||||
|
if isSlot(in) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
producer, ok := NodeForOutput(in)
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("node %s: input %q is neither a slot nor a known node output", n.Code, in)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if producer.Stage >= n.Stage {
|
||||||
|
t.Errorf("node %s (stage %d): input %q is produced by node %s at stage %d, want stage < %d",
|
||||||
|
n.Code, n.Stage, in, producer.Code, producer.Stage, n.Stage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEveryOutputHasExactlyOneProducer(t *testing.T) {
|
||||||
|
count := map[string]int{}
|
||||||
|
for _, n := range Nodes {
|
||||||
|
for _, o := range n.Outputs {
|
||||||
|
count[o]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, o := range Outputs {
|
||||||
|
if c := count[o.Code]; c != 1 {
|
||||||
|
t.Errorf("output %q has %d producers, want exactly 1", o.Code, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// No node should produce something that isn't in the Outputs list.
|
||||||
|
for code, c := range count {
|
||||||
|
if _, ok := OutputByCode(code); !ok {
|
||||||
|
t.Errorf("node output %q (%d producers) is not declared in Outputs", code, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStagesGroupingAndOrder(t *testing.T) {
|
||||||
|
stages := Stages()
|
||||||
|
if len(stages) != 4 {
|
||||||
|
t.Fatalf("got %d stages, want 4", len(stages))
|
||||||
|
}
|
||||||
|
want := [][]string{
|
||||||
|
{"00", "01", "02", "03"},
|
||||||
|
{"10"},
|
||||||
|
{"20", "21"},
|
||||||
|
{"30", "31"},
|
||||||
|
}
|
||||||
|
for i, group := range stages {
|
||||||
|
var codes []string
|
||||||
|
for _, n := range group {
|
||||||
|
codes = append(codes, n.Code)
|
||||||
|
}
|
||||||
|
if len(codes) != len(want[i]) {
|
||||||
|
t.Fatalf("stage group %d: got codes %v, want %v", i, codes, want[i])
|
||||||
|
}
|
||||||
|
for j, c := range codes {
|
||||||
|
if c != want[i][j] {
|
||||||
|
t.Errorf("stage group %d: got codes %v, want %v", i, codes, want[i])
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAllGroupCodesAreValidSlugs(t *testing.T) {
|
||||||
|
for _, s := range Slots {
|
||||||
|
if !slugRe.MatchString(s.Code) {
|
||||||
|
t.Errorf("slot code %q is not a valid slug", s.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, o := range Outputs {
|
||||||
|
if !slugRe.MatchString(o.Code) {
|
||||||
|
t.Errorf("output code %q is not a valid slug", o.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNodeByCode(t *testing.T) {
|
||||||
|
n, ok := NodeByCode("00")
|
||||||
|
if !ok || n.Name != "z_integrated" {
|
||||||
|
t.Fatalf("NodeByCode(00) = %+v, %v", n, ok)
|
||||||
|
}
|
||||||
|
if _, ok := NodeByCode("99"); ok {
|
||||||
|
t.Fatal("NodeByCode(99) should not be found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSlotByCode(t *testing.T) {
|
||||||
|
s, ok := SlotByCode(SlotZBiodiversity)
|
||||||
|
if !ok || s.Title != "Z Biodiversity" {
|
||||||
|
t.Fatalf("SlotByCode(%s) = %+v, %v", SlotZBiodiversity, s, ok)
|
||||||
|
}
|
||||||
|
if _, ok := SlotByCode("nope"); ok {
|
||||||
|
t.Fatal("SlotByCode(nope) should not be found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputByCode(t *testing.T) {
|
||||||
|
o, ok := OutputByCode(OutE)
|
||||||
|
if !ok || o.Title != "E Ecological Constraint" {
|
||||||
|
t.Fatalf("OutputByCode(%s) = %+v, %v", OutE, o, ok)
|
||||||
|
}
|
||||||
|
if _, ok := OutputByCode("nope"); ok {
|
||||||
|
t.Fatal("OutputByCode(nope) should not be found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNodeForOutput(t *testing.T) {
|
||||||
|
n, ok := NodeForOutput(OutV1)
|
||||||
|
if !ok || n.Code != "30" {
|
||||||
|
t.Fatalf("NodeForOutput(%s) = %+v, %v", OutV1, n, ok)
|
||||||
|
}
|
||||||
|
if _, ok := NodeForOutput("nope"); ok {
|
||||||
|
t.Fatal("NodeForOutput(nope) should not be found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Functional zoning ranks the ORIGINAL suitability rasters P1..P5 against E
|
||||||
|
// (reference notebook cells "Three scenarios" and zoning_core.py); the
|
||||||
|
// conflict layers C_k are diagnostics and must not replace P_k here.
|
||||||
|
func TestFuncZoningTakesEAndOriginalP(t *testing.T) {
|
||||||
|
n, ok := NodeByCode("30")
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("node 30 missing")
|
||||||
|
}
|
||||||
|
want := []string{OutE, SlotPIrrigation, SlotPSnow, SlotPCrop, SlotPPasture, SlotPTourism}
|
||||||
|
if len(n.Inputs) != len(want) {
|
||||||
|
t.Fatalf("node 30 inputs = %v, want %v", n.Inputs, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if n.Inputs[i] != want[i] {
|
||||||
|
t.Fatalf("node 30 input[%d] = %s, want %s (order is E, then sectors 1..5)", i, n.Inputs[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
215
internal/landscape/kernels.go
Normal file
215
internal/landscape/kernels.go
Normal file
@@ -0,0 +1,215 @@
|
|||||||
|
package landscape
|
||||||
|
|
||||||
|
// valid reports whether a class value is in the analysis domain (1..5). Any
|
||||||
|
// other value, including nodata and NaN, is invalid.
|
||||||
|
func valid(v float64) bool { return v >= 1 && v <= 5 }
|
||||||
|
|
||||||
|
// classify maps a float32 score onto classes 1..5 with the fixed intervals
|
||||||
|
// used by the reference notebook (comparisons in float32, like numpy on a
|
||||||
|
// float32 array). Scores outside [1, 5] yield NoData.
|
||||||
|
func classify(x float32) int16 {
|
||||||
|
switch {
|
||||||
|
case x < 1.0:
|
||||||
|
return NoData
|
||||||
|
case x <= 1.8:
|
||||||
|
return 1
|
||||||
|
case x <= 2.6:
|
||||||
|
return 2
|
||||||
|
case x <= 3.4:
|
||||||
|
return 3
|
||||||
|
case x <= 4.2:
|
||||||
|
return 4
|
||||||
|
case x <= 5.0:
|
||||||
|
return 5
|
||||||
|
default:
|
||||||
|
return NoData
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kernelZIntegrated: all four Z valid -> classify(0.25*Z1 + 0.25*Z2 + 0.25*Z3 + 0.25*Z4).
|
||||||
|
func kernelZIntegrated(in [][]float64, out [][]int16, _ Params) {
|
||||||
|
for i := range out[0] {
|
||||||
|
z1, z2, z3, z4 := in[0][i], in[1][i], in[2][i], in[3][i]
|
||||||
|
if !(valid(z1) && valid(z2) && valid(z3) && valid(z4)) {
|
||||||
|
out[0][i] = NoData
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// explicit conversions forbid FMA fusion so results match numpy's per-op rounding
|
||||||
|
score := float32(float32(0.25)*float32(z1)) + float32(float32(0.25)*float32(z2)) +
|
||||||
|
float32(float32(0.25)*float32(z3)) + float32(float32(0.25)*float32(z4))
|
||||||
|
out[0][i] = classify(score)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kernelUIntegrated: water and soil required, at least one of crop/veg;
|
||||||
|
// U = (0.25*water + 0.25*soil + [0.25*crop] + [0.25*veg]) / (0.5 + [0.25] + [0.25]).
|
||||||
|
func kernelUIntegrated(in [][]float64, out [][]int16, _ Params) {
|
||||||
|
for i := range out[0] {
|
||||||
|
water, soil, crop, veg := in[0][i], in[1][i], in[2][i], in[3][i]
|
||||||
|
cropOK, vegOK := valid(crop), valid(veg)
|
||||||
|
if !(valid(water) && valid(soil) && (cropOK || vegOK)) {
|
||||||
|
out[0][i] = NoData
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// explicit conversions forbid FMA fusion so results match numpy's per-op rounding
|
||||||
|
num := float32(float32(0.25)*float32(water)) + float32(float32(0.25)*float32(soil))
|
||||||
|
den := float32(0.5)
|
||||||
|
if cropOK {
|
||||||
|
num += float32(float32(0.25) * float32(crop))
|
||||||
|
den += 0.25
|
||||||
|
}
|
||||||
|
if vegOK {
|
||||||
|
num += float32(float32(0.25) * float32(veg))
|
||||||
|
den += 0.25
|
||||||
|
}
|
||||||
|
out[0][i] = classify(num / den)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kernelZMax: all four valid -> max.
|
||||||
|
func kernelZMax(in [][]float64, out [][]int16, _ Params) {
|
||||||
|
for i := range out[0] {
|
||||||
|
m, ok := int16(0), true
|
||||||
|
for _, layer := range in {
|
||||||
|
v := layer[i]
|
||||||
|
if !valid(v) {
|
||||||
|
ok = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if int16(v) > m {
|
||||||
|
m = int16(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
m = NoData
|
||||||
|
}
|
||||||
|
out[0][i] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kernelUMax: water and soil required plus one of crop/veg; max over valid layers.
|
||||||
|
func kernelUMax(in [][]float64, out [][]int16, _ Params) {
|
||||||
|
for i := range out[0] {
|
||||||
|
water, soil, crop, veg := in[0][i], in[1][i], in[2][i], in[3][i]
|
||||||
|
cropOK, vegOK := valid(crop), valid(veg)
|
||||||
|
if !(valid(water) && valid(soil) && (cropOK || vegOK)) {
|
||||||
|
out[0][i] = NoData
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m := max(int16(water), int16(soil))
|
||||||
|
if cropOK {
|
||||||
|
m = max(m, int16(crop))
|
||||||
|
}
|
||||||
|
if vegOK {
|
||||||
|
m = max(m, int16(veg))
|
||||||
|
}
|
||||||
|
out[0][i] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kernelE: Z and U valid -> max(Z, U).
|
||||||
|
func kernelE(in [][]float64, out [][]int16, _ Params) {
|
||||||
|
for i := range out[0] {
|
||||||
|
z, u := in[0][i], in[1][i]
|
||||||
|
if !(valid(z) && valid(u)) {
|
||||||
|
out[0][i] = NoData
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[0][i] = max(int16(z), int16(u))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kernelCConflicts: inputs P1..P5, E; output j = min(Pj, E) where both valid.
|
||||||
|
func kernelCConflicts(in [][]float64, out [][]int16, _ Params) {
|
||||||
|
e := in[5]
|
||||||
|
for j := 0; j < 5; j++ {
|
||||||
|
p := in[j]
|
||||||
|
for i := range out[j] {
|
||||||
|
if !(valid(p[i]) && valid(e[i])) {
|
||||||
|
out[j][i] = NoData
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[j][i] = min(int16(p[i]), int16(e[i]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kernelMaxAnyValid: max over the valid inputs; NoData when none is valid.
|
||||||
|
// Used for P_max and C_max.
|
||||||
|
func kernelMaxAnyValid(in [][]float64, out [][]int16, _ Params) {
|
||||||
|
for i := range out[0] {
|
||||||
|
m, any := int16(0), false
|
||||||
|
for _, layer := range in {
|
||||||
|
if v := layer[i]; valid(v) {
|
||||||
|
any = true
|
||||||
|
m = max(m, int16(v))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !any {
|
||||||
|
m = NoData
|
||||||
|
}
|
||||||
|
out[0][i] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kernelFuncZoning implements TZ_functional_zoning_go.md §4. Inputs: E, then
|
||||||
|
// S1..S5 (irrigation, snow, crop, pasture, tourism). Outputs: V1, V2, V3.
|
||||||
|
func kernelFuncZoning(in [][]float64, out [][]int16, p Params) {
|
||||||
|
e := in[0]
|
||||||
|
rules := [3]func(e, s int16) bool{
|
||||||
|
func(e, s int16) bool {
|
||||||
|
return (e <= 2 && s >= int16(p.V1MinSE12)) || (e == 3 && s >= int16(p.V1MinSE3))
|
||||||
|
},
|
||||||
|
func(e, s int16) bool {
|
||||||
|
return e <= 2 || (e == 3 && s >= int16(p.V2MinSE3)) || (e == 4 && s >= int16(p.V2MinSE4))
|
||||||
|
},
|
||||||
|
func(e, s int16) bool { return e <= 4 },
|
||||||
|
}
|
||||||
|
var s [5]int16
|
||||||
|
var base [5]bool
|
||||||
|
for i := range out[0] {
|
||||||
|
if !valid(e[i]) {
|
||||||
|
out[0][i], out[1][i], out[2][i] = NoData, NoData, NoData
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ev := int16(e[i])
|
||||||
|
anyBase := false
|
||||||
|
for k := 0; k < 5; k++ {
|
||||||
|
base[k] = valid(in[k+1][i])
|
||||||
|
if base[k] {
|
||||||
|
s[k] = int16(in[k+1][i])
|
||||||
|
anyBase = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !anyBase {
|
||||||
|
out[0][i], out[1][i], out[2][i] = NoData, NoData, NoData
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for v := 0; v < 3; v++ {
|
||||||
|
var cand [5]bool
|
||||||
|
maxS, anyCand := int16(0), false
|
||||||
|
for k := 0; k < 5; k++ {
|
||||||
|
cand[k] = base[k] && rules[v](ev, s[k])
|
||||||
|
if cand[k] {
|
||||||
|
anyCand = true
|
||||||
|
if s[k] > maxS {
|
||||||
|
maxS = s[k]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !anyCand {
|
||||||
|
out[v][i] = 1 // regulated
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sector := int16(0)
|
||||||
|
for _, code := range p.TiePriority {
|
||||||
|
k := code - 1
|
||||||
|
if cand[k] && s[k] == maxS {
|
||||||
|
sector = int16(code)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out[v][i] = sector + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
123
internal/landscape/kernels_test.go
Normal file
123
internal/landscape/kernels_test.go
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
package landscape
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func run(k Kernel, nOut int, in ...[]float64) [][]int16 {
|
||||||
|
out := make([][]int16, nOut)
|
||||||
|
for j := range out {
|
||||||
|
out[j] = make([]int16, len(in[0]))
|
||||||
|
}
|
||||||
|
k(in, out, DefaultParams())
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClassify(t *testing.T) {
|
||||||
|
cases := map[float32]int16{1.0: 1, 1.8: 1, 1.81: 2, 2.6: 2, 2.75: 3, 3.4: 3, 3.5: 4, 4.2: 4, 4.25: 5, 5.0: 5}
|
||||||
|
for x, want := range cases {
|
||||||
|
if got := classify(x); got != want {
|
||||||
|
t.Errorf("classify(%v)=%d want %d", x, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if classify(0.9) != NoData || classify(5.1) != NoData {
|
||||||
|
t.Error("out of range must be NoData")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKernelZIntegrated(t *testing.T) {
|
||||||
|
// pixel0: all valid (1,2,3,4) -> 2.5 -> class 2 ; pixel1: one nodata -> NoData ; pixel2: all 5 -> 5 -> class 5
|
||||||
|
got := run(kernelZIntegrated, 1,
|
||||||
|
[]float64{1, -9999, 5}, []float64{2, 2, 5}, []float64{3, 3, 5}, []float64{4, 4, 5})
|
||||||
|
if want := []int16{2, NoData, 5}; !reflect.DeepEqual(got[0], want) {
|
||||||
|
t.Fatalf("got %v want %v", got[0], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKernelUIntegrated_Applicability(t *testing.T) {
|
||||||
|
// order: water, soil, crop, veg
|
||||||
|
// p0: all valid 1,1,5,5 -> (0.25+0.25+1.25+1.25)/1.0 = 3.0 -> class 3
|
||||||
|
// p1: crop invalid: 1,1,-,5 -> (0.25+0.25+1.25)/0.75 = 2.333 -> class 2
|
||||||
|
// p2: crop and veg invalid -> NoData
|
||||||
|
// p3: water invalid -> NoData
|
||||||
|
got := run(kernelUIntegrated, 1,
|
||||||
|
[]float64{1, 1, 1, -9999}, []float64{1, 1, 1, 1}, []float64{5, 0, 0, 5}, []float64{5, 5, 0, 5})
|
||||||
|
if want := []int16{3, 2, NoData, NoData}; !reflect.DeepEqual(got[0], want) {
|
||||||
|
t.Fatalf("got %v want %v", got[0], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKernelZMax_RequiresAllValid(t *testing.T) {
|
||||||
|
got := run(kernelZMax, 1, []float64{1, 1}, []float64{4, 4}, []float64{2, -9999}, []float64{3, 3})
|
||||||
|
if want := []int16{4, NoData}; !reflect.DeepEqual(got[0], want) {
|
||||||
|
t.Fatalf("got %v want %v", got[0], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKernelUMax_IgnoresInvalidCropVeg(t *testing.T) {
|
||||||
|
// water, soil, crop, veg ; p0: 1,2,-,5 -> 5 ; p1: 3,2,0,0 -> NoData (needs crop or veg) ; p2: 3,2,1,0 -> 3
|
||||||
|
got := run(kernelUMax, 1, []float64{1, 3, 3}, []float64{2, 2, 2}, []float64{0, 0, 1}, []float64{5, 0, 0})
|
||||||
|
if want := []int16{5, NoData, 3}; !reflect.DeepEqual(got[0], want) {
|
||||||
|
t.Fatalf("got %v want %v", got[0], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKernelE(t *testing.T) {
|
||||||
|
got := run(kernelE, 1, []float64{2, 5, -9999}, []float64{4, 1, 3})
|
||||||
|
if want := []int16{4, 5, NoData}; !reflect.DeepEqual(got[0], want) {
|
||||||
|
t.Fatalf("got %v want %v", got[0], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKernelCConflicts(t *testing.T) {
|
||||||
|
// 5 P then E; p0: E=3 -> min(P,3); p1: E invalid -> all NoData; p2: P2 invalid
|
||||||
|
e := []float64{3, 0, 4}
|
||||||
|
got := run(kernelCConflicts, 5,
|
||||||
|
[]float64{5, 5, 5}, []float64{1, 1, 7}, []float64{3, 3, 3}, []float64{4, 4, 4}, []float64{2, 2, 2}, e)
|
||||||
|
want := [][]int16{{3, NoData, 4}, {1, NoData, NoData}, {3, NoData, 3}, {3, NoData, 4}, {2, NoData, 2}}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("got %v want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKernelMaxAnyValid(t *testing.T) {
|
||||||
|
got := run(kernelMaxAnyValid, 1, []float64{1, 0, 0}, []float64{4, 0, 2}, []float64{-9999, 0, 0})
|
||||||
|
if want := []int16{4, NoData, 2}; !reflect.DeepEqual(got[0], want) {
|
||||||
|
t.Fatalf("got %v want %v", got[0], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TZ §7 edge cases. Inputs: E, then C1..C5 (irrigation, snow, crop, pasture, tourism).
|
||||||
|
func TestKernelFuncZoning_TZEdgeCases(t *testing.T) {
|
||||||
|
e := []float64{0, 3, 5, 4, 2, 2, 3}
|
||||||
|
s1 := []float64{3, 0, 3, 5, 3, 2, 4}
|
||||||
|
s2 := []float64{3, 0, 3, 5, 3, 2, 4}
|
||||||
|
s3 := []float64{3, 0, 3, 2, 3, 2, 5}
|
||||||
|
s4 := []float64{3, 0, 3, 2, 3, 2, 1}
|
||||||
|
s5 := []float64{3, 0, 3, 2, 3, 2, 1}
|
||||||
|
got := run(kernelFuncZoning, 3, e, s1, s2, s3, s4, s5)
|
||||||
|
// pixel: 0 E invalid -> NoData ; 1 all S invalid -> NoData ; 2 E=5 -> regulated(1) everywhere ;
|
||||||
|
// 3 E=4, S=5 for sectors 1,2 -> V1 regulated, V2/V3 sector 1 -> class 2 ;
|
||||||
|
// 4 all S equal (3), E=2 -> irrigation (2) in all ; 5 E=2, S=2 -> V1 regulated (S<3), V2 best=irrigation (2), V3 irrigation (2)
|
||||||
|
// 6 E=3: S=[4,4,5,1,1] -> V1 (S>=4): winners 1,2,3 max 5 -> crop (4); V2 (S>=3) -> crop (4); V3 -> crop (4)
|
||||||
|
want := [][]int16{
|
||||||
|
{NoData, NoData, 1, 1, 2, 1, 4}, // V1
|
||||||
|
{NoData, NoData, 1, 2, 2, 2, 4}, // V2
|
||||||
|
{NoData, NoData, 1, 2, 2, 2, 4}, // V3
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("got %v want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKernelFuncZoning_TiePriorityIsConfigurable(t *testing.T) {
|
||||||
|
in := [][]float64{{2}, {3}, {3}, {3}, {3}, {3}}
|
||||||
|
out := [][]int16{{0}, {0}, {0}}
|
||||||
|
p := DefaultParams()
|
||||||
|
p.TiePriority = []int{5, 4, 3, 2, 1}
|
||||||
|
kernelFuncZoning(in, out, p)
|
||||||
|
if out[0][0] != 6 || out[1][0] != 6 || out[2][0] != 6 {
|
||||||
|
t.Fatalf("expected tourism (6) everywhere, got %v", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
81
internal/landscape/params.go
Normal file
81
internal/landscape/params.go
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
package landscape
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Params are the run-time knobs of the graph (TZ §6): window size, tie
|
||||||
|
// priority for functional zoning, and the V1/V2 suitability thresholds.
|
||||||
|
type Params struct {
|
||||||
|
WindowSize int `json:"window_size"`
|
||||||
|
TiePriority []int `json:"tie_priority"`
|
||||||
|
V1MinSE12 int `json:"v1_min_s_e12"`
|
||||||
|
V1MinSE3 int `json:"v1_min_s_e3"`
|
||||||
|
V2MinSE3 int `json:"v2_min_s_e3"`
|
||||||
|
V2MinSE4 int `json:"v2_min_s_e4"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultParams returns the TZ defaults.
|
||||||
|
func DefaultParams() Params {
|
||||||
|
return Params{WindowSize: 2048, TiePriority: []int{1, 2, 3, 4, 5}, V1MinSE12: 3, V1MinSE3: 4, V2MinSE3: 3, V2MinSE4: 5}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithDefaults fills zero-valued fields from DefaultParams.
|
||||||
|
func (p Params) WithDefaults() Params {
|
||||||
|
d := DefaultParams()
|
||||||
|
if p.WindowSize == 0 {
|
||||||
|
p.WindowSize = d.WindowSize
|
||||||
|
}
|
||||||
|
if len(p.TiePriority) == 0 {
|
||||||
|
p.TiePriority = d.TiePriority
|
||||||
|
}
|
||||||
|
if p.V1MinSE12 == 0 {
|
||||||
|
p.V1MinSE12 = d.V1MinSE12
|
||||||
|
}
|
||||||
|
if p.V1MinSE3 == 0 {
|
||||||
|
p.V1MinSE3 = d.V1MinSE3
|
||||||
|
}
|
||||||
|
if p.V2MinSE3 == 0 {
|
||||||
|
p.V2MinSE3 = d.V2MinSE3
|
||||||
|
}
|
||||||
|
if p.V2MinSE4 == 0 {
|
||||||
|
p.V2MinSE4 = d.V2MinSE4
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate checks ranges: window 256..8192, thresholds 1..5, tie_priority a
|
||||||
|
// permutation of 1..5.
|
||||||
|
func (p Params) Validate() error {
|
||||||
|
var errs []error
|
||||||
|
if p.WindowSize < 256 || p.WindowSize > 8192 {
|
||||||
|
errs = append(errs, fmt.Errorf("window_size must be in 256..8192"))
|
||||||
|
}
|
||||||
|
for name, v := range map[string]int{"v1_min_s_e12": p.V1MinSE12, "v1_min_s_e3": p.V1MinSE3, "v2_min_s_e3": p.V2MinSE3, "v2_min_s_e4": p.V2MinSE4} {
|
||||||
|
if v < 1 || v > 5 {
|
||||||
|
errs = append(errs, fmt.Errorf("%s must be in 1..5", name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seen := map[int]bool{}
|
||||||
|
for _, k := range p.TiePriority {
|
||||||
|
if k < 1 || k > 5 || seen[k] {
|
||||||
|
errs = append(errs, fmt.Errorf("tie_priority must be a permutation of 1..5"))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
seen[k] = true
|
||||||
|
}
|
||||||
|
if len(p.TiePriority) != 5 && len(seen) == len(p.TiePriority) {
|
||||||
|
errs = append(errs, fmt.Errorf("tie_priority must list all of 1..5"))
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ForNode keeps only the fields that influence the node's result (window size
|
||||||
|
// never does), so fingerprints do not change when irrelevant knobs change.
|
||||||
|
func (p Params) ForNode(code string) Params {
|
||||||
|
if code == "30" {
|
||||||
|
return Params{TiePriority: p.TiePriority, V1MinSE12: p.V1MinSE12, V1MinSE3: p.V1MinSE3, V2MinSE3: p.V2MinSE3, V2MinSE4: p.V2MinSE4}
|
||||||
|
}
|
||||||
|
return Params{}
|
||||||
|
}
|
||||||
58
internal/landscape/params_test.go
Normal file
58
internal/landscape/params_test.go
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
package landscape
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDefaultParamsValidate(t *testing.T) {
|
||||||
|
if err := DefaultParams().Validate(); err != nil {
|
||||||
|
t.Fatalf("DefaultParams().Validate() = %v, want nil", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWithDefaultsFillsZeroFields(t *testing.T) {
|
||||||
|
got := Params{}.WithDefaults()
|
||||||
|
want := DefaultParams()
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("Params{}.WithDefaults() = %+v, want %+v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidate_TiePriorityDuplicateInvalid(t *testing.T) {
|
||||||
|
p := DefaultParams()
|
||||||
|
p.TiePriority = []int{1, 1, 2, 3, 4}
|
||||||
|
if err := p.Validate(); err == nil {
|
||||||
|
t.Fatal("expected error for tie_priority with duplicate values")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidate_WindowSizeOutOfRangeInvalid(t *testing.T) {
|
||||||
|
p := DefaultParams()
|
||||||
|
p.WindowSize = 100
|
||||||
|
if err := p.Validate(); err == nil {
|
||||||
|
t.Fatal("expected error for window_size below 256")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForNode_NonFuncZoningNodeIsZero(t *testing.T) {
|
||||||
|
p := DefaultParams()
|
||||||
|
got := p.ForNode("00")
|
||||||
|
if !reflect.DeepEqual(got, Params{}) {
|
||||||
|
t.Fatalf("ForNode(00) = %+v, want zero value", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForNode_FuncZoningHasNoWindowSize(t *testing.T) {
|
||||||
|
p := DefaultParams()
|
||||||
|
got := p.ForNode("30")
|
||||||
|
if got.WindowSize != 0 {
|
||||||
|
t.Fatalf("ForNode(30).WindowSize = %d, want 0", got.WindowSize)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got.TiePriority, p.TiePriority) {
|
||||||
|
t.Fatalf("ForNode(30).TiePriority = %v, want %v", got.TiePriority, p.TiePriority)
|
||||||
|
}
|
||||||
|
if got.V1MinSE12 != p.V1MinSE12 || got.V1MinSE3 != p.V1MinSE3 || got.V2MinSE3 != p.V2MinSE3 || got.V2MinSE4 != p.V2MinSE4 {
|
||||||
|
t.Fatalf("ForNode(30) thresholds = %+v, want matching %+v", got, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
94
internal/messaging/rabbitmq/landscape_consumer.go
Normal file
94
internal/messaging/rabbitmq/landscape_consumer.go
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
package rabbitmq
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// LandscapeRunRoutingKey routes Landscape Program run jobs.
|
||||||
|
LandscapeRunRoutingKey = "landscape.run"
|
||||||
|
// LandscapeRunQueue receives Landscape Program run jobs.
|
||||||
|
LandscapeRunQueue = "gis.landscape.run"
|
||||||
|
// markRunFailedTimeout bounds the best-effort terminal status write made
|
||||||
|
// after Execute failed, on a context detached from the delivery's.
|
||||||
|
markRunFailedTimeout = 10 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// LandscapeJob is the message body for a Landscape Program run job.
|
||||||
|
type LandscapeJob struct {
|
||||||
|
RunID uuid.UUID `json:"run_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LandscapeJobPublisher publishes Landscape Program run jobs; it implements
|
||||||
|
// service.LandscapeJobEnqueuer.
|
||||||
|
type LandscapeJobPublisher struct {
|
||||||
|
pub *Publisher
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLandscapeJobPublisher returns a LandscapeJobPublisher.
|
||||||
|
func NewLandscapeJobPublisher(pub *Publisher) *LandscapeJobPublisher {
|
||||||
|
return &LandscapeJobPublisher{pub: pub}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnqueueRun publishes an execution job for the given run.
|
||||||
|
func (p *LandscapeJobPublisher) EnqueueRun(ctx context.Context, runID uuid.UUID) error {
|
||||||
|
body, err := json.Marshal(LandscapeJob{RunID: runID})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return p.pub.Publish(ctx, LandscapeRunRoutingKey, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LandscapeProcessor executes Landscape Program runs; implemented by the service.
|
||||||
|
type LandscapeProcessor interface {
|
||||||
|
Execute(ctx context.Context, runID uuid.UUID) error
|
||||||
|
// MarkRunFailed records a terminal failure on the run so it does not remain
|
||||||
|
// stuck in an in-progress status.
|
||||||
|
MarkRunFailed(ctx context.Context, runID uuid.UUID, reason string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// landscapeHandler executes one Landscape Program run per delivery.
|
||||||
|
type landscapeHandler struct {
|
||||||
|
proc LandscapeProcessor
|
||||||
|
log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h landscapeHandler) Handle(ctx context.Context, d amqp.Delivery) error {
|
||||||
|
var job LandscapeJob
|
||||||
|
if err := json.Unmarshal(d.Body, &job); err != nil {
|
||||||
|
return fmt.Errorf("decode landscape run job: %w", err)
|
||||||
|
}
|
||||||
|
h.log.Info("processing landscape run", "run_id", job.RunID)
|
||||||
|
|
||||||
|
err := h.proc.Execute(ctx, job.RunID)
|
||||||
|
if err != nil {
|
||||||
|
// The delivery is nacked without requeue (see Consumer.dispatch), so the
|
||||||
|
// run will not be retried. Record the failure on the run itself;
|
||||||
|
// otherwise it stays "running" forever. This covers the errors Execute
|
||||||
|
// returns before it can mark the run itself (a missing run, a failed
|
||||||
|
// status write), since Execute already marks the run failed for every
|
||||||
|
// node failure. Best-effort: log if marking fails, but still return the
|
||||||
|
// original error so the delivery is nacked.
|
||||||
|
// Detached from ctx: Execute may well have failed *because* ctx was
|
||||||
|
// cancelled (shutdown), and the write must still land or the run stays
|
||||||
|
// "running" forever.
|
||||||
|
mctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), markRunFailedTimeout)
|
||||||
|
if markErr := h.proc.MarkRunFailed(mctx, job.RunID, err.Error()); markErr != nil {
|
||||||
|
h.log.Error("mark landscape run failed", "run_id", job.RunID, "error", markErr)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLandscapeRunHandler returns a handler that executes Landscape Program runs.
|
||||||
|
func NewLandscapeRunHandler(p LandscapeProcessor, log *slog.Logger) Handler {
|
||||||
|
return landscapeHandler{proc: p, log: log}
|
||||||
|
}
|
||||||
90
internal/messaging/rabbitmq/landscape_consumer_test.go
Normal file
90
internal/messaging/rabbitmq/landscape_consumer_test.go
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
package rabbitmq
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeLandscapeProcessor records MarkRunFailed calls and returns a preset error
|
||||||
|
// from Execute.
|
||||||
|
type fakeLandscapeProcessor struct {
|
||||||
|
runErr error
|
||||||
|
executed []uuid.UUID
|
||||||
|
failedID uuid.UUID
|
||||||
|
failedWith string
|
||||||
|
failedCnt int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeLandscapeProcessor) Execute(_ context.Context, runID uuid.UUID) error {
|
||||||
|
f.executed = append(f.executed, runID)
|
||||||
|
return f.runErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeLandscapeProcessor) MarkRunFailed(_ context.Context, runID uuid.UUID, reason string) error {
|
||||||
|
f.failedCnt++
|
||||||
|
f.failedID = runID
|
||||||
|
f.failedWith = reason
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func landscapeDelivery(t *testing.T, id uuid.UUID) amqp.Delivery {
|
||||||
|
t.Helper()
|
||||||
|
body, err := json.Marshal(LandscapeJob{RunID: id})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal job: %v", err)
|
||||||
|
}
|
||||||
|
return amqp.Delivery{Body: body}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLandscapeHandler_MarksRunFailedOnExecuteError(t *testing.T) {
|
||||||
|
id := uuid.New()
|
||||||
|
fp := &fakeLandscapeProcessor{runErr: errors.New("node 00: grid mismatch")}
|
||||||
|
h := NewLandscapeRunHandler(fp, discardLogger())
|
||||||
|
|
||||||
|
err := h.Handle(context.Background(), landscapeDelivery(t, id))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected the run error to be returned so the delivery is nacked")
|
||||||
|
}
|
||||||
|
if len(fp.executed) != 1 || fp.executed[0] != id {
|
||||||
|
t.Fatalf("Execute called with %v, want [%s]", fp.executed, id)
|
||||||
|
}
|
||||||
|
if fp.failedCnt != 1 {
|
||||||
|
t.Fatalf("expected MarkRunFailed to be called once, got %d", fp.failedCnt)
|
||||||
|
}
|
||||||
|
if fp.failedID != id {
|
||||||
|
t.Fatalf("MarkRunFailed called with wrong id: got %s want %s", fp.failedID, id)
|
||||||
|
}
|
||||||
|
if fp.failedWith != "node 00: grid mismatch" {
|
||||||
|
t.Fatalf("MarkRunFailed called with wrong reason: %q", fp.failedWith)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLandscapeHandler_DoesNotMarkFailedOnSuccess(t *testing.T) {
|
||||||
|
fp := &fakeLandscapeProcessor{runErr: nil}
|
||||||
|
h := NewLandscapeRunHandler(fp, discardLogger())
|
||||||
|
|
||||||
|
if err := h.Handle(context.Background(), landscapeDelivery(t, uuid.New())); err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if fp.failedCnt != 0 {
|
||||||
|
t.Fatalf("expected MarkRunFailed not to be called on success, got %d", fp.failedCnt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLandscapeHandler_UndecodableBodyIsNacked(t *testing.T) {
|
||||||
|
fp := &fakeLandscapeProcessor{}
|
||||||
|
h := NewLandscapeRunHandler(fp, discardLogger())
|
||||||
|
|
||||||
|
err := h.Handle(context.Background(), amqp.Delivery{Body: []byte("not json")})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected a decode error so the delivery is nacked")
|
||||||
|
}
|
||||||
|
if len(fp.executed) != 0 {
|
||||||
|
t.Fatalf("Execute must not run for an undecodable body, got %v", fp.executed)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -58,12 +58,15 @@ func (c *GDALConverter) Footprint(ctx context.Context, src string) ([]byte, erro
|
|||||||
return info.Wgs84Extent, nil
|
return info.Wgs84Extent, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// VectorGeometry reads every feature of a vector file and returns their combined
|
// VectorFeatures reads every feature of a vector file and returns their
|
||||||
// geometry as a GeoJSON GeometryCollection reprojected to EPSG:4326, or nil if
|
// geometries, reprojected to EPSG:4326, in file order (nil if the file has no
|
||||||
// the file has no features. The caller (PostGIS) dissolves the collection into
|
// features). A feature with no geometry yields a nil entry at its index, so
|
||||||
// the union of all features. Zipped ESRI shapefiles are read in place via GDAL's
|
// the result stays aligned with attribute rows read from the same file (the
|
||||||
// /vsizip/ virtual filesystem; GeoJSON and GeoPackage are read directly.
|
// shapefile format itself has no other way to pair a .shp geometry record with
|
||||||
func (c *GDALConverter) VectorGeometry(ctx context.Context, src string) ([]byte, error) {
|
// its .dbf attribute row than matching by position). Zipped ESRI shapefiles
|
||||||
|
// are read in place via GDAL's /vsizip/ virtual filesystem; GeoJSON and
|
||||||
|
// GeoPackage are read directly.
|
||||||
|
func (c *GDALConverter) VectorFeatures(ctx context.Context, src string) ([]json.RawMessage, error) {
|
||||||
input := src
|
input := src
|
||||||
if strings.EqualFold(filepath.Ext(src), ".zip") {
|
if strings.EqualFold(filepath.Ext(src), ".zip") {
|
||||||
input = "/vsizip/" + src
|
input = "/vsizip/" + src
|
||||||
@@ -89,20 +92,16 @@ func (c *GDALConverter) VectorGeometry(ctx context.Context, src string) ([]byte,
|
|||||||
if err := json.Unmarshal(out, &fc); err != nil {
|
if err := json.Unmarshal(out, &fc); err != nil {
|
||||||
return nil, fmt.Errorf("parse ogr2ogr output: %w", err)
|
return nil, fmt.Errorf("parse ogr2ogr output: %w", err)
|
||||||
}
|
}
|
||||||
|
if len(fc.Features) == 0 {
|
||||||
geoms := make([]json.RawMessage, 0, len(fc.Features))
|
|
||||||
for _, f := range fc.Features {
|
|
||||||
if len(f.Geometry) == 0 || string(f.Geometry) == "null" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
geoms = append(geoms, f.Geometry)
|
|
||||||
}
|
|
||||||
if len(geoms) == 0 {
|
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return json.Marshal(struct {
|
geoms := make([]json.RawMessage, len(fc.Features))
|
||||||
Type string `json:"type"`
|
for i, f := range fc.Features {
|
||||||
Geometries []json.RawMessage `json:"geometries"`
|
if len(f.Geometry) == 0 || string(f.Geometry) == "null" {
|
||||||
}{Type: "GeometryCollection", Geometries: geoms})
|
continue
|
||||||
|
}
|
||||||
|
geoms[i] = f.Geometry
|
||||||
|
}
|
||||||
|
return geoms, nil
|
||||||
}
|
}
|
||||||
|
|||||||
140
internal/raster/godal.go
Normal file
140
internal/raster/godal.go
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
package raster
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/airbusgeo/godal"
|
||||||
|
)
|
||||||
|
|
||||||
|
var registerOnce sync.Once
|
||||||
|
|
||||||
|
// GodalIO implements IO with the GDAL bindings (github.com/airbusgeo/godal).
|
||||||
|
// GDAL datasets are not goroutine-safe: callers must not share a Reader or
|
||||||
|
// Writer between goroutines without external synchronisation.
|
||||||
|
type GodalIO struct{}
|
||||||
|
|
||||||
|
// NewGodalIO registers GDAL drivers (once) and returns a GodalIO.
|
||||||
|
func NewGodalIO() *GodalIO {
|
||||||
|
registerOnce.Do(godal.RegisterAll)
|
||||||
|
return &GodalIO{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultWriterOptions returns the Landscape Program output profile.
|
||||||
|
func DefaultWriterOptions() WriterOptions {
|
||||||
|
return WriterOptions{NoData: -9999, TileSize: 512, Compression: "DEFLATE"}
|
||||||
|
}
|
||||||
|
|
||||||
|
type godalReader struct {
|
||||||
|
ds *godal.Dataset
|
||||||
|
grid Grid
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GodalIO) Open(_ context.Context, path string) (Reader, error) {
|
||||||
|
ds, err := godal.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open raster %s: %w", path, err)
|
||||||
|
}
|
||||||
|
st := ds.Structure()
|
||||||
|
if st.NBands < 1 {
|
||||||
|
ds.Close()
|
||||||
|
return nil, fmt.Errorf("open raster %s: no bands", path)
|
||||||
|
}
|
||||||
|
gt, err := ds.GeoTransform()
|
||||||
|
if err != nil {
|
||||||
|
ds.Close()
|
||||||
|
return nil, fmt.Errorf("open raster %s: geotransform: %w", path, err)
|
||||||
|
}
|
||||||
|
return &godalReader{ds: ds, grid: Grid{Width: st.SizeX, Height: st.SizeY, GeoTransform: gt, ProjectionWKT: ds.Projection()}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *godalReader) Grid() Grid { return r.grid }
|
||||||
|
|
||||||
|
func (r *godalReader) NoData() (float64, bool) { return r.ds.Bands()[0].NoData() }
|
||||||
|
|
||||||
|
func (r *godalReader) ReadFloat64(w Window, dst []float64) error {
|
||||||
|
if len(dst) < w.Width*w.Height {
|
||||||
|
return fmt.Errorf("read window: dst has %d elements, need %d", len(dst), w.Width*w.Height)
|
||||||
|
}
|
||||||
|
return r.ds.Bands()[0].Read(w.X, w.Y, dst[:w.Width*w.Height], w.Width, w.Height)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *godalReader) Close() error { return r.ds.Close() }
|
||||||
|
|
||||||
|
type godalWriter struct{ ds *godal.Dataset }
|
||||||
|
|
||||||
|
func (g *GodalIO) Create(_ context.Context, path string, grid Grid, o WriterOptions) (Writer, error) {
|
||||||
|
return createGTiff(path, grid, o, "IF_SAFER")
|
||||||
|
}
|
||||||
|
|
||||||
|
// createGTiff writes a tiled Int16 GeoTIFF. bigTIFF is the value of the
|
||||||
|
// BIGTIFF creation option ("IF_SAFER" in production); it is a parameter so
|
||||||
|
// tests can force a BigTIFF header.
|
||||||
|
func createGTiff(path string, grid Grid, o WriterOptions, bigTIFF string) (Writer, error) {
|
||||||
|
tile := fmt.Sprint(o.TileSize)
|
||||||
|
ds, err := godal.Create(godal.GTiff, path, 1, godal.Int16, grid.Width, grid.Height,
|
||||||
|
godal.CreationOption("TILED=YES", "BLOCKXSIZE="+tile, "BLOCKYSIZE="+tile,
|
||||||
|
"COMPRESS="+o.Compression, "PREDICTOR=2", "BIGTIFF="+bigTIFF))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("create raster %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if err := ds.SetGeoTransform(grid.GeoTransform); err != nil {
|
||||||
|
ds.Close()
|
||||||
|
return nil, fmt.Errorf("create raster %s: geotransform: %w", path, err)
|
||||||
|
}
|
||||||
|
if grid.ProjectionWKT != "" {
|
||||||
|
if err := ds.SetProjection(grid.ProjectionWKT); err != nil {
|
||||||
|
ds.Close()
|
||||||
|
return nil, fmt.Errorf("create raster %s: projection: %w", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ds.Bands()[0].SetNoData(float64(o.NoData)); err != nil {
|
||||||
|
ds.Close()
|
||||||
|
return nil, fmt.Errorf("create raster %s: nodata: %w", path, err)
|
||||||
|
}
|
||||||
|
return &godalWriter{ds: ds}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *godalWriter) WriteInt16(win Window, src []int16) error {
|
||||||
|
if len(src) < win.Width*win.Height {
|
||||||
|
return fmt.Errorf("write window: src has %d elements, need %d", len(src), win.Width*win.Height)
|
||||||
|
}
|
||||||
|
return w.ds.Bands()[0].Write(win.X, win.Y, src[:win.Width*win.Height], win.Width, win.Height)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *godalWriter) Close() error { return w.ds.Close() }
|
||||||
|
|
||||||
|
// SameGrid reports whether two grids describe the same pixels: equal size,
|
||||||
|
// geotransform within 1e-9 (absolute), and the same CRS (GDAL IsSame; an
|
||||||
|
// empty projection only matches an empty projection).
|
||||||
|
//
|
||||||
|
// ProjectionWKT is parsed with godal.NewSpatialRef rather than
|
||||||
|
// NewSpatialRefFromWKT: despite the field name, callers may pass any user
|
||||||
|
// projection string GDAL's SetProjection accepts (e.g. "EPSG:32642"), not
|
||||||
|
// only WKT, and NewSpatialRefFromWKT rejects non-WKT input.
|
||||||
|
func (g *GodalIO) SameGrid(a, b Grid) bool {
|
||||||
|
if a.Width != b.Width || a.Height != b.Height {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range a.GeoTransform {
|
||||||
|
if math.Abs(a.GeoTransform[i]-b.GeoTransform[i]) > 1e-9 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a.ProjectionWKT == "" || b.ProjectionWKT == "" {
|
||||||
|
return a.ProjectionWKT == b.ProjectionWKT
|
||||||
|
}
|
||||||
|
sa, err := godal.NewSpatialRef(a.ProjectionWKT)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer sa.Close()
|
||||||
|
sb, err := godal.NewSpatialRef(b.ProjectionWKT)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer sb.Close()
|
||||||
|
return sa.IsSame(sb)
|
||||||
|
}
|
||||||
103
internal/raster/godal_test.go
Normal file
103
internal/raster/godal_test.go
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
package raster
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gis/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGodalIO_RoundTrip(t *testing.T) {
|
||||||
|
rio := NewGodalIO()
|
||||||
|
ctx := context.Background()
|
||||||
|
path := filepath.Join(t.TempDir(), "out.tif")
|
||||||
|
grid := Grid{Width: 10, Height: 7, GeoTransform: [6]float64{100, 30, 0, 200, 0, -30}, ProjectionWKT: "EPSG:32642"}
|
||||||
|
|
||||||
|
w, err := rio.Create(ctx, path, grid, DefaultWriterOptions())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
full := make([]int16, 10*7)
|
||||||
|
for i := range full {
|
||||||
|
full[i] = int16(i%6) - 1 // -1..4 (-1 stands in for garbage, not nodata)
|
||||||
|
}
|
||||||
|
full[0] = -9999
|
||||||
|
if err := w.WriteInt16(Window{0, 0, 10, 7}, full); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := w.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
r, err := rio.Open(ctx, path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
g := r.Grid()
|
||||||
|
if g.Width != 10 || g.Height != 7 || g.GeoTransform != grid.GeoTransform || g.ProjectionWKT == "" {
|
||||||
|
t.Fatalf("grid mismatch: %+v", g)
|
||||||
|
}
|
||||||
|
if nd, ok := r.NoData(); !ok || nd != -9999 {
|
||||||
|
t.Fatalf("nodata = %v %v", nd, ok)
|
||||||
|
}
|
||||||
|
dst := make([]float64, 3*2)
|
||||||
|
if err := r.ReadFloat64(Window{X: 8, Y: 5, Width: 2, Height: 2}, dst[:4]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// pixel (8,5) has index 58 -> 58%6-1 = 3 ; (9,5)=59 -> 4 ; (8,6)=68 -> 1 ; (9,6)=69 -> 2
|
||||||
|
want := []float64{3, 4, 1, 2}
|
||||||
|
for i, v := range want {
|
||||||
|
if dst[i] != v {
|
||||||
|
t.Fatalf("pixel %d = %v want %v", i, dst[i], v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !rio.SameGrid(g, grid) {
|
||||||
|
t.Fatal("SameGrid should be true for identical grids")
|
||||||
|
}
|
||||||
|
other := grid
|
||||||
|
other.GeoTransform[0] += 1
|
||||||
|
if rio.SameGrid(g, other) {
|
||||||
|
t.Fatal("SameGrid should be false for shifted origin")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGodalIO_BigTIFFPassesContentValidation guards the upload magic-byte
|
||||||
|
// check against the pipeline's own output: GDAL writes a BigTIFF header
|
||||||
|
// (version 43) whenever BIGTIFF is YES or IF_SAFER kicks in for a large
|
||||||
|
// raster, and domain.ValidateFileContent must accept it.
|
||||||
|
func TestGodalIO_BigTIFFPassesContentValidation(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "big.tif")
|
||||||
|
grid := Grid{Width: 8, Height: 8, GeoTransform: [6]float64{100, 30, 0, 200, 0, -30}, ProjectionWKT: "EPSG:32642"}
|
||||||
|
|
||||||
|
w, err := createGTiff(path, grid, DefaultWriterOptions(), "YES")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := w.WriteInt16(Window{0, 0, 8, 8}, make([]int16, 8*8)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := w.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
head := make([]byte, 4)
|
||||||
|
if _, err := io.ReadFull(f, head); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.HasPrefix(head, []byte("II+\x00")) && !bytes.HasPrefix(head, []byte("MM\x00+")) {
|
||||||
|
t.Fatalf("BIGTIFF=YES did not produce a BigTIFF header: % x", head)
|
||||||
|
}
|
||||||
|
if err := domain.ValidateFileContent(".tif", head); err != nil {
|
||||||
|
t.Fatalf("ValidateFileContent rejected our own BigTIFF output: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
77
internal/raster/io.go
Normal file
77
internal/raster/io.go
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
package raster
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// Grid describes a raster's pixel grid: its size, geotransform, and
|
||||||
|
// projection. Two grids describe the same pixels when IO.SameGrid reports
|
||||||
|
// true for them.
|
||||||
|
type Grid struct {
|
||||||
|
Width, Height int
|
||||||
|
GeoTransform [6]float64
|
||||||
|
ProjectionWKT string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Window is a rectangular block of pixels within a raster, in pixel/line
|
||||||
|
// coordinates with (0,0) at the top-left corner.
|
||||||
|
type Window struct {
|
||||||
|
X, Y, Width, Height int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reader reads pixel data and metadata from an existing raster.
|
||||||
|
type Reader interface {
|
||||||
|
// Grid returns the raster's pixel grid.
|
||||||
|
Grid() Grid
|
||||||
|
// NoData returns the raster's nodata value and whether one is set.
|
||||||
|
NoData() (float64, bool)
|
||||||
|
// ReadFloat64 reads the pixels in w into dst, row-major, converting the
|
||||||
|
// underlying pixel type to float64. len(dst) must be at least
|
||||||
|
// w.Width*w.Height.
|
||||||
|
ReadFloat64(w Window, dst []float64) error
|
||||||
|
// Close releases the underlying dataset handle.
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriterOptions configures a raster created by IO.Create.
|
||||||
|
type WriterOptions struct {
|
||||||
|
// NoData is the nodata value written to the output band.
|
||||||
|
NoData int16
|
||||||
|
// TileSize is the internal tile size (both block dimensions) in pixels.
|
||||||
|
TileSize int
|
||||||
|
// Compression is the GDAL COMPRESS creation option value.
|
||||||
|
Compression string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Writer writes pixel data to a raster created by IO.Create.
|
||||||
|
type Writer interface {
|
||||||
|
// WriteInt16 writes src, row-major, into the pixels described by w.
|
||||||
|
// len(src) must be at least w.Width*w.Height.
|
||||||
|
WriteInt16(w Window, src []int16) error
|
||||||
|
// Close flushes and releases the underlying dataset handle.
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// IO is an engine-agnostic block I/O abstraction over raster storage. It lets
|
||||||
|
// callers open and create rasters without depending on a particular GDAL
|
||||||
|
// binding.
|
||||||
|
type IO interface {
|
||||||
|
// Open opens the raster at path for reading.
|
||||||
|
Open(ctx context.Context, path string) (Reader, error)
|
||||||
|
// Create creates a new single-band int16 raster at path with the given
|
||||||
|
// grid and writer options, ready for WriteInt16 calls.
|
||||||
|
Create(ctx context.Context, path string, g Grid, o WriterOptions) (Writer, error)
|
||||||
|
// SameGrid reports whether a and b describe the same pixel grid: equal
|
||||||
|
// size, geotransform, and coordinate reference system.
|
||||||
|
SameGrid(a, b Grid) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Windows tiles a w×h raster into row-major windows of at most size×size.
|
||||||
|
func Windows(w, h, size int) []Window {
|
||||||
|
var out []Window
|
||||||
|
for y := 0; y < h; y += size {
|
||||||
|
wh := min(size, h-y)
|
||||||
|
for x := 0; x < w; x += size {
|
||||||
|
out = append(out, Window{X: x, Y: y, Width: min(size, w-x), Height: wh})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
61
internal/raster/io_test.go
Normal file
61
internal/raster/io_test.go
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
package raster
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWindows(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
w, h, size int
|
||||||
|
want []Window
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "10x10 size4 tiles into 9 windows",
|
||||||
|
w: 10, h: 10, size: 4,
|
||||||
|
want: []Window{
|
||||||
|
{X: 0, Y: 0, Width: 4, Height: 4},
|
||||||
|
{X: 4, Y: 0, Width: 4, Height: 4},
|
||||||
|
{X: 8, Y: 0, Width: 2, Height: 4},
|
||||||
|
{X: 0, Y: 4, Width: 4, Height: 4},
|
||||||
|
{X: 4, Y: 4, Width: 4, Height: 4},
|
||||||
|
{X: 8, Y: 4, Width: 2, Height: 4},
|
||||||
|
{X: 0, Y: 8, Width: 4, Height: 2},
|
||||||
|
{X: 4, Y: 8, Width: 4, Height: 2},
|
||||||
|
{X: 8, Y: 8, Width: 2, Height: 2},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "5x3 size8 fits in one window",
|
||||||
|
w: 5, h: 3, size: 8,
|
||||||
|
want: []Window{
|
||||||
|
{X: 0, Y: 0, Width: 5, Height: 3},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := Windows(tt.w, tt.h, tt.size)
|
||||||
|
if !reflect.DeepEqual(got, tt.want) {
|
||||||
|
t.Fatalf("Windows(%d, %d, %d) = %+v, want %+v", tt.w, tt.h, tt.size, got, tt.want)
|
||||||
|
}
|
||||||
|
if len(got) > 0 {
|
||||||
|
last := got[len(got)-1]
|
||||||
|
wantLast := tt.want[len(tt.want)-1]
|
||||||
|
if last != wantLast {
|
||||||
|
t.Fatalf("last window = %+v, want %+v", last, wantLast)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultWriterOptions(t *testing.T) {
|
||||||
|
got := DefaultWriterOptions()
|
||||||
|
want := WriterOptions{NoData: -9999, TileSize: 512, Compression: "DEFLATE"}
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("DefaultWriterOptions() = %+v, want %+v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -27,20 +27,22 @@ func NewDatasetRepository(pool *pgxpool.Pool) *DatasetRepository {
|
|||||||
// datasetColumns lists the dataset columns for SELECT and RETURNING. The
|
// datasetColumns lists the dataset columns for SELECT and RETURNING. The
|
||||||
// geometry is exposed as GeoJSON (jsonb) rather than its raw EWKB form, and a
|
// geometry is exposed as GeoJSON (jsonb) rather than its raw EWKB form, and a
|
||||||
// bounding box array is derived for raster datasets only.
|
// bounding box array is derived for raster datasets only.
|
||||||
const datasetColumns = `id, category_id, name, description, unit, filename, storage_key, cog_storage_key, file_type, size_bytes, content_type, properties, meta, automated, status, attribute_columns, kato_column, year_columns, parse_error, ST_AsGeoJSON(geometry)::jsonb AS geometry,
|
const datasetColumns = `id, category_id, dg_id, to_char(start_date, 'YYYY-MM-DD') AS start_date, to_char(end_date, 'YYYY-MM-DD') AS end_date, name, description, unit, filename, storage_key, cog_storage_key, file_type, size_bytes, content_type, properties, meta, automated, status, attribute_columns, kato_column, year_columns, parse_error, ST_AsGeoJSON(geometry)::jsonb AS geometry,
|
||||||
CASE WHEN file_type = 'raster' AND geometry IS NOT NULL
|
CASE WHEN file_type = 'raster' AND geometry IS NOT NULL
|
||||||
THEN ARRAY[ST_XMin(geometry), ST_YMin(geometry), ST_XMax(geometry), ST_YMax(geometry)]
|
THEN ARRAY[ST_XMin(geometry), ST_YMin(geometry), ST_XMax(geometry), ST_YMax(geometry)]
|
||||||
ELSE NULL END AS bbox,
|
ELSE NULL END AS bbox,
|
||||||
|
geojson, sld_storage_key, sld_filename, sld_content_type,
|
||||||
created_at, updated_at`
|
created_at, updated_at`
|
||||||
|
|
||||||
func scanDataset(row pgx.Row) (domain.Dataset, error) {
|
func scanDataset(row pgx.Row) (domain.Dataset, error) {
|
||||||
var d domain.Dataset
|
var d domain.Dataset
|
||||||
err := row.Scan(
|
err := row.Scan(
|
||||||
&d.ID, &d.CategoryID, &d.Name, &d.Description, &d.Unit,
|
&d.ID, &d.CategoryID, &d.DatasetGroupID, &d.StartDate, &d.EndDate, &d.Name, &d.Description, &d.Unit,
|
||||||
&d.Filename, &d.StorageKey, &d.CogStorageKey, &d.FileType, &d.SizeBytes, &d.ContentType,
|
&d.Filename, &d.StorageKey, &d.CogStorageKey, &d.FileType, &d.SizeBytes, &d.ContentType,
|
||||||
&d.Properties, &d.Meta, &d.Automated, &d.Status,
|
&d.Properties, &d.Meta, &d.Automated, &d.Status,
|
||||||
&d.AttributeColumns, &d.KatoColumn, &d.YearColumns, &d.ParseError,
|
&d.AttributeColumns, &d.KatoColumn, &d.YearColumns, &d.ParseError,
|
||||||
&d.Geometry, &d.BBox, &d.CreatedAt, &d.UpdatedAt,
|
&d.Geometry, &d.BBox, &d.GeoJSON, &d.SldStorageKey, &d.SldFilename, &d.SldContentType,
|
||||||
|
&d.CreatedAt, &d.UpdatedAt,
|
||||||
)
|
)
|
||||||
return d, err
|
return d, err
|
||||||
}
|
}
|
||||||
@@ -57,11 +59,28 @@ func nullableJSON(raw json.RawMessage) any {
|
|||||||
// Create inserts a new dataset and returns the stored row.
|
// Create inserts a new dataset and returns the stored row.
|
||||||
func (r *DatasetRepository) Create(ctx context.Context, d domain.Dataset) (domain.Dataset, error) {
|
func (r *DatasetRepository) Create(ctx context.Context, d domain.Dataset) (domain.Dataset, error) {
|
||||||
row := r.pool.QueryRow(ctx,
|
row := r.pool.QueryRow(ctx,
|
||||||
`INSERT INTO datasets (category_id, name, description, unit, filename, storage_key, file_type, size_bytes, content_type, properties, meta, automated, status)
|
`INSERT INTO datasets (category_id, name, description, unit, filename, storage_key, file_type, size_bytes, content_type, properties, meta, automated, status, dg_id, start_date, end_date)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15::date, $16::date)
|
||||||
RETURNING `+datasetColumns,
|
RETURNING `+datasetColumns,
|
||||||
d.CategoryID, d.Name, d.Description, d.Unit, d.Filename, d.StorageKey, d.FileType, d.SizeBytes, d.ContentType,
|
d.CategoryID, d.Name, d.Description, d.Unit, d.Filename, d.StorageKey, d.FileType, d.SizeBytes, d.ContentType,
|
||||||
nullableJSON(d.Properties), nullableJSON(d.Meta), d.Automated, d.Status,
|
nullableJSON(d.Properties), nullableJSON(d.Meta), d.Automated, d.Status,
|
||||||
|
d.DatasetGroupID, d.StartDate, d.EndDate,
|
||||||
|
)
|
||||||
|
out, err := scanDataset(row)
|
||||||
|
return out, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateFields overwrites a dataset's editable metadata columns. Nil pointers
|
||||||
|
// clear the nullable columns. Returns domain.ErrNotFound if the dataset does
|
||||||
|
// not exist, or domain.ErrConflict if the category/group does not exist.
|
||||||
|
func (r *DatasetRepository) UpdateFields(ctx context.Context, id uuid.UUID, f domain.DatasetFields) (domain.Dataset, error) {
|
||||||
|
row := r.pool.QueryRow(ctx,
|
||||||
|
`UPDATE datasets
|
||||||
|
SET category_id = $2, name = $3, description = $4, unit = $5,
|
||||||
|
dg_id = $6, start_date = $7::date, end_date = $8::date, updated_at = now()
|
||||||
|
WHERE id = $1
|
||||||
|
RETURNING `+datasetColumns,
|
||||||
|
id, f.CategoryID, f.Name, f.Description, f.Unit, f.DatasetGroupID, f.StartDate, f.EndDate,
|
||||||
)
|
)
|
||||||
out, err := scanDataset(row)
|
out, err := scanDataset(row)
|
||||||
return out, mapError(err)
|
return out, mapError(err)
|
||||||
@@ -193,6 +212,55 @@ func (r *DatasetRepository) MarkReady(ctx context.Context, id uuid.UUID, geometr
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetGeoJSON stores the pre-assembled GeoJSON FeatureCollection for a dataset
|
||||||
|
// (nil -> NULL). It is written at processing time and served verbatim by the
|
||||||
|
// .geojson endpoint.
|
||||||
|
func (r *DatasetRepository) SetGeoJSON(ctx context.Context, id uuid.UUID, geojson []byte) error {
|
||||||
|
tag, err := r.pool.Exec(ctx,
|
||||||
|
`UPDATE datasets SET geojson = $2, updated_at = now() WHERE id = $1`,
|
||||||
|
id, nullableJSON(json.RawMessage(geojson)),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return mapError(err)
|
||||||
|
}
|
||||||
|
if tag.RowsAffected() == 0 {
|
||||||
|
return domain.ErrNotFound
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetSLD stores the dataset's SLD (Styled Layer Descriptor) style file
|
||||||
|
// metadata, replacing any previous one, and returns the updated row.
|
||||||
|
func (r *DatasetRepository) SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error) {
|
||||||
|
row := r.pool.QueryRow(ctx,
|
||||||
|
`UPDATE datasets
|
||||||
|
SET sld_storage_key = $2, sld_filename = $3, sld_content_type = $4, updated_at = now()
|
||||||
|
WHERE id = $1
|
||||||
|
RETURNING `+datasetColumns,
|
||||||
|
id, key, filename, contentType,
|
||||||
|
)
|
||||||
|
out, err := scanDataset(row)
|
||||||
|
return out, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClearGeometryAndGeoJSON nulls out a dataset's stored geometry and persisted
|
||||||
|
// GeoJSON. Used before re-enqueuing a dataset's processing job so a stale
|
||||||
|
// (possibly incorrect) geometry/GeoJSON is never served while the job is
|
||||||
|
// pending; the async worker repopulates both once it completes.
|
||||||
|
func (r *DatasetRepository) ClearGeometryAndGeoJSON(ctx context.Context, id uuid.UUID) error {
|
||||||
|
tag, err := r.pool.Exec(ctx,
|
||||||
|
`UPDATE datasets SET geometry = NULL, geojson = NULL, updated_at = now() WHERE id = $1`,
|
||||||
|
id,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return mapError(err)
|
||||||
|
}
|
||||||
|
if tag.RowsAffected() == 0 {
|
||||||
|
return domain.ErrNotFound
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// ReplaceObservations atomically replaces all observations for a dataset.
|
// ReplaceObservations atomically replaces all observations for a dataset.
|
||||||
func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID uuid.UUID, obs []domain.Observation) error {
|
func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID uuid.UUID, obs []domain.Observation) error {
|
||||||
tx, err := r.pool.Begin(ctx)
|
tx, err := r.pool.Begin(ctx)
|
||||||
@@ -208,15 +276,23 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u
|
|||||||
if len(obs) > 0 {
|
if len(obs) > 0 {
|
||||||
rows := make([][]any, len(obs))
|
rows := make([][]any, len(obs))
|
||||||
for i, o := range obs {
|
for i, o := range obs {
|
||||||
d, err := time.Parse("2006-01-02", o.Date)
|
d, err := time.Parse("2006-01-02", o.StartDate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("invalid observation date %q: %w", o.Date, err)
|
return fmt.Errorf("invalid observation start_date %q: %w", o.StartDate, err)
|
||||||
}
|
}
|
||||||
rows[i] = []any{datasetID, o.KatoCode, d, o.Value, o.ValueText}
|
var endDate any
|
||||||
|
if o.EndDate != nil {
|
||||||
|
dt, err := time.Parse("2006-01-02", *o.EndDate)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid observation end_date %q: %w", *o.EndDate, err)
|
||||||
|
}
|
||||||
|
endDate = dt
|
||||||
|
}
|
||||||
|
rows[i] = []any{datasetID, o.KatoCode, d, endDate, o.Value, o.ValueText}
|
||||||
}
|
}
|
||||||
_, err := tx.CopyFrom(ctx,
|
_, err := tx.CopyFrom(ctx,
|
||||||
pgx.Identifier{"dataset_observations"},
|
pgx.Identifier{"dataset_observations"},
|
||||||
[]string{"dataset_id", "kato_code", "date", "value", "value_text"},
|
[]string{"dataset_id", "kato_code", "start_date", "end_date", "value", "value_text"},
|
||||||
pgx.CopyFromRows(rows),
|
pgx.CopyFromRows(rows),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -227,16 +303,16 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u
|
|||||||
return mapError(tx.Commit(ctx))
|
return mapError(tx.Commit(ctx))
|
||||||
}
|
}
|
||||||
|
|
||||||
const observationColumns = `id, dataset_id, kato_code, to_char(date, 'YYYY-MM-DD') AS date, value, value_text`
|
const observationColumns = `id, dataset_id, kato_code, to_char(start_date, 'YYYY-MM-DD') AS start_date, to_char(end_date, 'YYYY-MM-DD') AS end_date, value, value_text`
|
||||||
|
|
||||||
func scanObservation(row pgx.Row) (domain.Observation, error) {
|
func scanObservation(row pgx.Row) (domain.Observation, error) {
|
||||||
var o domain.Observation
|
var o domain.Observation
|
||||||
err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.Date, &o.Value, &o.ValueText)
|
err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.StartDate, &o.EndDate, &o.Value, &o.ValueText)
|
||||||
return o, err
|
return o, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListObservations returns a page of observations for a dataset, optionally
|
// ListObservations returns a page of observations for a dataset, optionally
|
||||||
// filtered by KATO code, ordered by (kato_code, date).
|
// filtered by KATO code, ordered by (kato_code, start_date).
|
||||||
func (r *DatasetRepository) ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error) {
|
func (r *DatasetRepository) ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error) {
|
||||||
base := `SELECT ` + observationColumns + ` FROM dataset_observations WHERE dataset_id = $1`
|
base := `SELECT ` + observationColumns + ` FROM dataset_observations WHERE dataset_id = $1`
|
||||||
|
|
||||||
@@ -246,11 +322,11 @@ func (r *DatasetRepository) ListObservations(ctx context.Context, datasetID uuid
|
|||||||
)
|
)
|
||||||
if katoCode != nil {
|
if katoCode != nil {
|
||||||
rows, err = r.pool.Query(ctx,
|
rows, err = r.pool.Query(ctx,
|
||||||
base+` AND kato_code = $2 ORDER BY kato_code, date LIMIT $3 OFFSET $4`,
|
base+` AND kato_code = $2 ORDER BY kato_code, start_date LIMIT $3 OFFSET $4`,
|
||||||
datasetID, *katoCode, limit, offset)
|
datasetID, *katoCode, limit, offset)
|
||||||
} else {
|
} else {
|
||||||
rows, err = r.pool.Query(ctx,
|
rows, err = r.pool.Query(ctx,
|
||||||
base+` ORDER BY kato_code, date LIMIT $2 OFFSET $3`,
|
base+` ORDER BY kato_code, start_date LIMIT $2 OFFSET $3`,
|
||||||
datasetID, limit, offset)
|
datasetID, limit, offset)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -292,12 +368,25 @@ func (r *DatasetRepository) GetByID(ctx context.Context, id uuid.UUID) (domain.D
|
|||||||
return out, mapError(err)
|
return out, mapError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
const datasetSummaryColumns = `id, category_id, name, description, unit, file_type, size_bytes, status, created_at, updated_at`
|
// LatestInGroup returns the freshest dataset of a group: greatest end_date
|
||||||
|
// (NULL end_date sorts last), ties broken by created_at DESC. status and
|
||||||
|
// fileType optionally restrict candidates. Returns domain.ErrNotFound when no
|
||||||
|
// dataset matches.
|
||||||
|
func (r *DatasetRepository) LatestInGroup(ctx context.Context, groupID uuid.UUID, status *string, fileType *domain.FileType) (domain.Dataset, error) {
|
||||||
|
where, args := datasetFilterClause(domain.DatasetFilter{DatasetGroupID: &groupID, Status: status, FileType: fileType}, nil)
|
||||||
|
row := r.pool.QueryRow(ctx,
|
||||||
|
`SELECT `+datasetColumns+` FROM datasets`+where+
|
||||||
|
` ORDER BY end_date DESC NULLS LAST, created_at DESC LIMIT 1`, args...)
|
||||||
|
out, err := scanDataset(row)
|
||||||
|
return out, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
const datasetSummaryColumns = `id, category_id, dg_id, to_char(start_date, 'YYYY-MM-DD') AS start_date, to_char(end_date, 'YYYY-MM-DD') AS end_date, name, description, unit, file_type, size_bytes, status, created_at, updated_at`
|
||||||
|
|
||||||
func scanDatasetSummary(row pgx.Row) (domain.DatasetSummary, error) {
|
func scanDatasetSummary(row pgx.Row) (domain.DatasetSummary, error) {
|
||||||
var d domain.DatasetSummary
|
var d domain.DatasetSummary
|
||||||
err := row.Scan(
|
err := row.Scan(
|
||||||
&d.ID, &d.CategoryID, &d.Name, &d.Description, &d.Unit,
|
&d.ID, &d.CategoryID, &d.DatasetGroupID, &d.StartDate, &d.EndDate, &d.Name, &d.Description, &d.Unit,
|
||||||
&d.FileType, &d.SizeBytes, &d.Status, &d.CreatedAt, &d.UpdatedAt,
|
&d.FileType, &d.SizeBytes, &d.Status, &d.CreatedAt, &d.UpdatedAt,
|
||||||
)
|
)
|
||||||
return d, err
|
return d, err
|
||||||
@@ -324,6 +413,22 @@ func datasetFilterClause(f domain.DatasetFilter, args []any) (string, []any) {
|
|||||||
args = append(args, *f.Status)
|
args = append(args, *f.Status)
|
||||||
conds = append(conds, fmt.Sprintf("status = $%d", len(args)))
|
conds = append(conds, fmt.Sprintf("status = $%d", len(args)))
|
||||||
}
|
}
|
||||||
|
if f.DatasetGroupID != nil {
|
||||||
|
args = append(args, *f.DatasetGroupID)
|
||||||
|
conds = append(conds, fmt.Sprintf("dg_id = $%d", len(args)))
|
||||||
|
}
|
||||||
|
for _, b := range []struct {
|
||||||
|
col, op string
|
||||||
|
val *string
|
||||||
|
}{
|
||||||
|
{"start_date", ">=", f.StartDateFrom}, {"start_date", "<=", f.StartDateTo},
|
||||||
|
{"end_date", ">=", f.EndDateFrom}, {"end_date", "<=", f.EndDateTo},
|
||||||
|
} {
|
||||||
|
if b.val != nil {
|
||||||
|
args = append(args, *b.val)
|
||||||
|
conds = append(conds, fmt.Sprintf("%s %s $%d::date", b.col, b.op, len(args)))
|
||||||
|
}
|
||||||
|
}
|
||||||
if len(conds) == 0 {
|
if len(conds) == 0 {
|
||||||
return "", args
|
return "", args
|
||||||
}
|
}
|
||||||
|
|||||||
115
internal/repository/postgres/dataset_db_test.go
Normal file
115
internal/repository/postgres/dataset_db_test.go
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
package postgres
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gis/internal/domain"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestDatasetRepository_GroupsPeriodsAndLatest exercises the Postgres-only
|
||||||
|
// parts of the dataset/group repositories added with dataset periods: the
|
||||||
|
// ::date casts on *string parameters, the date filter clause, and the
|
||||||
|
// "freshest in group" ordering (end_date DESC NULLS LAST, created_at DESC).
|
||||||
|
func TestDatasetRepository_GroupsPeriodsAndLatest(t *testing.T) {
|
||||||
|
pool := testPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
groups := NewDatasetGroupRepository(pool)
|
||||||
|
datasets := NewDatasetRepository(pool)
|
||||||
|
categories := NewCategoryRepository(pool)
|
||||||
|
|
||||||
|
suffix := uuid.NewString()[:8]
|
||||||
|
cat, err := categories.Create(ctx, domain.Category{Code: "dbtest-" + suffix, Name: "db test"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create category: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM categories WHERE id = $1`, cat.ID) })
|
||||||
|
|
||||||
|
group, err := groups.Create(ctx, domain.DatasetGroup{Code: "dbtest-group-" + suffix, Name: "db test group"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create group: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM dataset_groups WHERE id = $1`, group.ID) })
|
||||||
|
|
||||||
|
s := func(v string) *string { return &v }
|
||||||
|
mk := func(name string, start, end *string) domain.Dataset {
|
||||||
|
d, err := datasets.Create(ctx, domain.Dataset{
|
||||||
|
CategoryID: cat.ID, DatasetGroupID: &group.ID, Name: name, Filename: name + ".tif",
|
||||||
|
StorageKey: "dbtest/" + suffix + "/" + name, FileType: domain.FileTypeRaster,
|
||||||
|
ContentType: "image/tiff", Status: domain.DatasetStatusReady,
|
||||||
|
StartDate: start, EndDate: end,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create dataset %s: %v", name, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM datasets WHERE id = $1`, d.ID) })
|
||||||
|
time.Sleep(5 * time.Millisecond) // distinct created_at for the tie-break
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
old := mk("old", s("2019-01-01"), s("2019-12-31"))
|
||||||
|
fresh := mk("fresh", s("2020-01-01"), s("2020-12-31"))
|
||||||
|
undated := mk("undated", nil, nil)
|
||||||
|
|
||||||
|
if old.StartDate == nil || *old.StartDate != "2019-01-01" || *old.EndDate != "2019-12-31" || old.DatasetGroupID == nil {
|
||||||
|
t.Fatalf("dates/group not round-tripped: %+v", old)
|
||||||
|
}
|
||||||
|
|
||||||
|
latest, err := datasets.LatestInGroup(ctx, group.ID, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LatestInGroup: %v", err)
|
||||||
|
}
|
||||||
|
if latest.ID != fresh.ID {
|
||||||
|
t.Fatalf("latest = %s, want fresh %s (undated %s must sort last)", latest.Name, fresh.ID, undated.ID)
|
||||||
|
}
|
||||||
|
ft := domain.FileTypeVector
|
||||||
|
if _, err := datasets.LatestInGroup(ctx, group.ID, nil, &ft); !errors.Is(err, domain.ErrNotFound) {
|
||||||
|
t.Fatalf("no vector in group: err = %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
page, err := datasets.ListSummaries(ctx, domain.DatasetFilter{
|
||||||
|
DatasetGroupID: &group.ID, EndDateFrom: s("2020-01-01"), EndDateTo: s("2020-12-31"),
|
||||||
|
}, 10, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListSummaries with date bounds: %v", err)
|
||||||
|
}
|
||||||
|
if len(page) != 1 || page[0].ID != fresh.ID {
|
||||||
|
t.Fatalf("date filter returned %+v, want only fresh", page)
|
||||||
|
}
|
||||||
|
n, err := datasets.Count(ctx, domain.DatasetFilter{DatasetGroupID: &group.ID, StartDateFrom: s("2019-06-01")})
|
||||||
|
if err != nil || n != 1 {
|
||||||
|
t.Fatalf("Count(start_date_from) = %d, %v; want 1", n, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
updated, err := datasets.UpdateFields(ctx, undated.ID, domain.DatasetFields{
|
||||||
|
CategoryID: cat.ID, Name: "undated-now-dated", DatasetGroupID: &group.ID,
|
||||||
|
StartDate: s("2021-01-01"), EndDate: s("2021-12-31"),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("UpdateFields: %v", err)
|
||||||
|
}
|
||||||
|
if updated.EndDate == nil || *updated.EndDate != "2021-12-31" {
|
||||||
|
t.Fatalf("UpdateFields did not store the period: %+v", updated)
|
||||||
|
}
|
||||||
|
if _, err := datasets.UpdateFields(ctx, undated.ID, domain.DatasetFields{
|
||||||
|
CategoryID: cat.ID, Name: "bad", StartDate: s("2021-12-31"), EndDate: s("2021-01-01"),
|
||||||
|
}); !errors.Is(err, domain.ErrValidation) {
|
||||||
|
t.Fatalf("reversed period must hit the CHECK constraint: err = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := groups.Delete(ctx, group.ID); !errors.Is(err, domain.ErrConflict) {
|
||||||
|
t.Fatalf("deleting a referenced group: err = %v, want ErrConflict", err)
|
||||||
|
}
|
||||||
|
g2, err := groups.SetSLD(ctx, group.ID, "k", "style.sld", "application/xml")
|
||||||
|
if err != nil || g2.SldStorageKey == nil || *g2.SldStorageKey != "k" {
|
||||||
|
t.Fatalf("SetSLD: %+v, %v", g2, err)
|
||||||
|
}
|
||||||
|
byCode, err := groups.GetByCode(ctx, group.Code)
|
||||||
|
if err != nil || byCode.ID != group.ID {
|
||||||
|
t.Fatalf("GetByCode: %+v, %v", byCode, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
117
internal/repository/postgres/dataset_group.go
Normal file
117
internal/repository/postgres/dataset_group.go
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
package postgres
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gis/internal/domain"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DatasetGroupRepository persists dataset groups in Postgres.
|
||||||
|
type DatasetGroupRepository struct{ pool *pgxpool.Pool }
|
||||||
|
|
||||||
|
// NewDatasetGroupRepository returns a DatasetGroupRepository backed by pool.
|
||||||
|
func NewDatasetGroupRepository(pool *pgxpool.Pool) *DatasetGroupRepository {
|
||||||
|
return &DatasetGroupRepository{pool: pool}
|
||||||
|
}
|
||||||
|
|
||||||
|
const datasetGroupColumns = `id, code, name, description, sld_storage_key, sld_filename, sld_content_type, created_at, updated_at`
|
||||||
|
|
||||||
|
func scanDatasetGroup(row pgx.Row) (domain.DatasetGroup, error) {
|
||||||
|
var g domain.DatasetGroup
|
||||||
|
err := row.Scan(&g.ID, &g.Code, &g.Name, &g.Description, &g.SldStorageKey, &g.SldFilename, &g.SldContentType, &g.CreatedAt, &g.UpdatedAt)
|
||||||
|
return g, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create inserts a new dataset group and returns the stored row.
|
||||||
|
func (r *DatasetGroupRepository) Create(ctx context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) {
|
||||||
|
row := r.pool.QueryRow(ctx,
|
||||||
|
`INSERT INTO dataset_groups (code, name, description) VALUES ($1, $2, $3) RETURNING `+datasetGroupColumns,
|
||||||
|
g.Code, g.Name, g.Description)
|
||||||
|
out, err := scanDatasetGroup(row)
|
||||||
|
return out, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetByID returns the group with the given id.
|
||||||
|
func (r *DatasetGroupRepository) GetByID(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, error) {
|
||||||
|
out, err := scanDatasetGroup(r.pool.QueryRow(ctx, `SELECT `+datasetGroupColumns+` FROM dataset_groups WHERE id = $1`, id))
|
||||||
|
return out, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetByCode returns the group with the given (unique) code.
|
||||||
|
func (r *DatasetGroupRepository) GetByCode(ctx context.Context, code string) (domain.DatasetGroup, error) {
|
||||||
|
out, err := scanDatasetGroup(r.pool.QueryRow(ctx, `SELECT `+datasetGroupColumns+` FROM dataset_groups WHERE code = $1`, code))
|
||||||
|
return out, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// List returns a page of groups ordered by code; code (exact) is optional.
|
||||||
|
func (r *DatasetGroupRepository) List(ctx context.Context, code *string, limit, offset int) ([]domain.DatasetGroup, error) {
|
||||||
|
where, args := groupWhere(code)
|
||||||
|
args = append(args, limit, offset)
|
||||||
|
rows, err := r.pool.Query(ctx, fmt.Sprintf(`SELECT %s FROM dataset_groups%s ORDER BY code LIMIT $%d OFFSET $%d`,
|
||||||
|
datasetGroupColumns, where, len(args)-1, len(args)), args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapError(err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
out := make([]domain.DatasetGroup, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
g, err := scanDatasetGroup(rows)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapError(err)
|
||||||
|
}
|
||||||
|
out = append(out, g)
|
||||||
|
}
|
||||||
|
return out, mapError(rows.Err())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns how many groups match the optional code filter.
|
||||||
|
func (r *DatasetGroupRepository) Count(ctx context.Context, code *string) (int, error) {
|
||||||
|
where, args := groupWhere(code)
|
||||||
|
var n int
|
||||||
|
err := r.pool.QueryRow(ctx, `SELECT count(*) FROM dataset_groups`+where, args...).Scan(&n)
|
||||||
|
return n, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupWhere builds the optional exact-code WHERE clause and its arguments.
|
||||||
|
func groupWhere(code *string) (string, []any) {
|
||||||
|
if code == nil {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
return " WHERE code = $1", []any{*code}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update replaces a group's editable fields and returns the stored row.
|
||||||
|
func (r *DatasetGroupRepository) Update(ctx context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) {
|
||||||
|
row := r.pool.QueryRow(ctx,
|
||||||
|
`UPDATE dataset_groups SET code = $2, name = $3, description = $4, updated_at = now() WHERE id = $1 RETURNING `+datasetGroupColumns,
|
||||||
|
g.ID, g.Code, g.Name, g.Description)
|
||||||
|
out, err := scanDatasetGroup(row)
|
||||||
|
return out, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes a group. Returns domain.ErrNotFound if absent and
|
||||||
|
// domain.ErrConflict (via the FK) if datasets still reference it.
|
||||||
|
func (r *DatasetGroupRepository) Delete(ctx context.Context, id uuid.UUID) error {
|
||||||
|
tag, err := r.pool.Exec(ctx, `DELETE FROM dataset_groups WHERE id = $1`, id)
|
||||||
|
if err != nil {
|
||||||
|
return mapError(err)
|
||||||
|
}
|
||||||
|
if tag.RowsAffected() == 0 {
|
||||||
|
return domain.ErrNotFound
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetSLD points the group at a stored SLD style object.
|
||||||
|
func (r *DatasetGroupRepository) SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.DatasetGroup, error) {
|
||||||
|
row := r.pool.QueryRow(ctx,
|
||||||
|
`UPDATE dataset_groups SET sld_storage_key = $2, sld_filename = $3, sld_content_type = $4, updated_at = now() WHERE id = $1 RETURNING `+datasetGroupColumns,
|
||||||
|
id, key, filename, contentType)
|
||||||
|
out, err := scanDatasetGroup(row)
|
||||||
|
return out, mapError(err)
|
||||||
|
}
|
||||||
@@ -9,14 +9,14 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// ListAllObservations returns every observation for a dataset ordered by
|
// ListAllObservations returns every observation for a dataset ordered by
|
||||||
// (kato_code, date). Unlike ListObservations it is unpaginated: it backs GeoJSON
|
// (kato_code, start_date). Unlike ListObservations it is unpaginated: it backs GeoJSON
|
||||||
// assembly, where all of a dataset's observations are needed at once.
|
// assembly, where all of a dataset's observations are needed at once.
|
||||||
func (r *DatasetRepository) ListAllObservations(ctx context.Context, datasetID uuid.UUID) ([]domain.Observation, error) {
|
func (r *DatasetRepository) ListAllObservations(ctx context.Context, datasetID uuid.UUID) ([]domain.Observation, error) {
|
||||||
rows, err := r.pool.Query(ctx,
|
rows, err := r.pool.Query(ctx,
|
||||||
`SELECT `+observationColumns+`
|
`SELECT `+observationColumns+`
|
||||||
FROM dataset_observations
|
FROM dataset_observations
|
||||||
WHERE dataset_id = $1
|
WHERE dataset_id = $1
|
||||||
ORDER BY kato_code, date`,
|
ORDER BY kato_code, start_date`,
|
||||||
datasetID)
|
datasetID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapError(err)
|
return nil, mapError(err)
|
||||||
|
|||||||
239
internal/repository/postgres/landscape.go
Normal file
239
internal/repository/postgres/landscape.go
Normal file
@@ -0,0 +1,239 @@
|
|||||||
|
package postgres
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
|
||||||
|
"gis/internal/domain"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LandscapeRepository persists Landscape Program runs in Postgres.
|
||||||
|
type LandscapeRepository struct {
|
||||||
|
pool *pgxpool.Pool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLandscapeRepository returns a LandscapeRepository backed by the given pool.
|
||||||
|
func NewLandscapeRepository(pool *pgxpool.Pool) *LandscapeRepository {
|
||||||
|
return &LandscapeRepository{pool: pool}
|
||||||
|
}
|
||||||
|
|
||||||
|
const runColumns = `id, status, from_stage, params, overrides, error, created_at, started_at, finished_at`
|
||||||
|
|
||||||
|
// scanRun scans a run row. overrides is stored as jsonb but scanned into
|
||||||
|
// []byte and unmarshalled manually so an empty/NULL value maps to an empty
|
||||||
|
// map rather than nil.
|
||||||
|
func scanRun(row pgx.Row) (domain.LPRun, error) {
|
||||||
|
var r domain.LPRun
|
||||||
|
var overrides []byte
|
||||||
|
err := row.Scan(&r.ID, &r.Status, &r.FromStage, &r.Params, &overrides, &r.Error, &r.CreatedAt, &r.StartedAt, &r.FinishedAt)
|
||||||
|
if err != nil {
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
r.Overrides = map[string]uuid.UUID{}
|
||||||
|
if len(overrides) > 0 {
|
||||||
|
if err := json.Unmarshal(overrides, &r.Overrides); err != nil {
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const nodeColumns = `id, run_id, node_code, stage, status, fingerprint, input_datasets, output_datasets, error, started_at, finished_at`
|
||||||
|
|
||||||
|
func scanNode(row pgx.Row) (domain.LPRunNode, error) {
|
||||||
|
var n domain.LPRunNode
|
||||||
|
err := row.Scan(&n.ID, &n.RunID, &n.NodeCode, &n.Stage, &n.Status, &n.Fingerprint, &n.Inputs, &n.Outputs, &n.Error, &n.StartedAt, &n.FinishedAt)
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreateRun inserts a new run and its nodes (status pending) in one
|
||||||
|
// transaction, and returns the run with its nodes populated (in the order
|
||||||
|
// they were given).
|
||||||
|
func (r *LandscapeRepository) CreateRun(ctx context.Context, run domain.LPRun) (domain.LPRun, error) {
|
||||||
|
tx, err := r.pool.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return domain.LPRun{}, mapError(err)
|
||||||
|
}
|
||||||
|
defer tx.Rollback(ctx)
|
||||||
|
|
||||||
|
overrides := run.Overrides
|
||||||
|
if overrides == nil {
|
||||||
|
overrides = map[string]uuid.UUID{}
|
||||||
|
}
|
||||||
|
overridesJSON, err := json.Marshal(overrides)
|
||||||
|
if err != nil {
|
||||||
|
return domain.LPRun{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
row := tx.QueryRow(ctx,
|
||||||
|
`INSERT INTO lp_runs (status, from_stage, params, overrides)
|
||||||
|
VALUES ($1, $2, $3, $4)
|
||||||
|
RETURNING `+runColumns,
|
||||||
|
run.Status, run.FromStage, run.Params, overridesJSON,
|
||||||
|
)
|
||||||
|
out, err := scanRun(row)
|
||||||
|
if err != nil {
|
||||||
|
return domain.LPRun{}, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes := make([]domain.LPRunNode, 0, len(run.Nodes))
|
||||||
|
for _, n := range run.Nodes {
|
||||||
|
row := tx.QueryRow(ctx,
|
||||||
|
`INSERT INTO lp_run_nodes (run_id, node_code, stage, status)
|
||||||
|
VALUES ($1, $2, $3, $4)
|
||||||
|
RETURNING `+nodeColumns,
|
||||||
|
out.ID, n.NodeCode, n.Stage, domain.LPNodePending,
|
||||||
|
)
|
||||||
|
node, err := scanNode(row)
|
||||||
|
if err != nil {
|
||||||
|
return domain.LPRun{}, mapError(err)
|
||||||
|
}
|
||||||
|
nodes = append(nodes, node)
|
||||||
|
}
|
||||||
|
out.Nodes = nodes
|
||||||
|
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return domain.LPRun{}, mapError(err)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetRun returns the run with the given id, with its nodes ordered by
|
||||||
|
// node_code. Returns domain.ErrNotFound if the run does not exist.
|
||||||
|
func (r *LandscapeRepository) GetRun(ctx context.Context, id uuid.UUID) (domain.LPRun, error) {
|
||||||
|
row := r.pool.QueryRow(ctx, `SELECT `+runColumns+` FROM lp_runs WHERE id = $1`, id)
|
||||||
|
out, err := scanRun(row)
|
||||||
|
if err != nil {
|
||||||
|
return domain.LPRun{}, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := r.pool.Query(ctx,
|
||||||
|
`SELECT `+nodeColumns+` FROM lp_run_nodes WHERE run_id = $1 ORDER BY node_code`, id)
|
||||||
|
if err != nil {
|
||||||
|
return domain.LPRun{}, mapError(err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
nodes := make([]domain.LPRunNode, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
n, err := scanNode(rows)
|
||||||
|
if err != nil {
|
||||||
|
return domain.LPRun{}, mapError(err)
|
||||||
|
}
|
||||||
|
nodes = append(nodes, n)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return domain.LPRun{}, mapError(err)
|
||||||
|
}
|
||||||
|
out.Nodes = nodes
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListRuns returns a page of runs (without nodes), newest first.
|
||||||
|
func (r *LandscapeRepository) ListRuns(ctx context.Context, limit, offset int) ([]domain.LPRun, error) {
|
||||||
|
rows, err := r.pool.Query(ctx,
|
||||||
|
`SELECT `+runColumns+` FROM lp_runs ORDER BY created_at DESC LIMIT $1 OFFSET $2`, limit, offset)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapError(err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
runs := make([]domain.LPRun, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
run, err := scanRun(rows)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapError(err)
|
||||||
|
}
|
||||||
|
runs = append(runs, run)
|
||||||
|
}
|
||||||
|
return runs, mapError(rows.Err())
|
||||||
|
}
|
||||||
|
|
||||||
|
// CountRuns returns the total number of runs.
|
||||||
|
func (r *LandscapeRepository) CountRuns(ctx context.Context) (int, error) {
|
||||||
|
var n int
|
||||||
|
err := r.pool.QueryRow(ctx, `SELECT count(*) FROM lp_runs`).Scan(&n)
|
||||||
|
return n, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetRunStatus updates a run's status and error. Transitioning to running
|
||||||
|
// sets started_at; transitioning to succeeded or failed sets finished_at.
|
||||||
|
// Returns domain.ErrNotFound if the run does not exist.
|
||||||
|
func (r *LandscapeRepository) SetRunStatus(ctx context.Context, id uuid.UUID, status domain.LPRunStatus, errMsg *string) error {
|
||||||
|
tag, err := r.pool.Exec(ctx,
|
||||||
|
// $2 is used both as the assigned value (varchar column) and in the CASE
|
||||||
|
// comparisons (text literals); the explicit ::text casts keep Postgres
|
||||||
|
// from deducing two different types for one parameter (SQLSTATE 42P08).
|
||||||
|
`UPDATE lp_runs SET status = $2::text, error = $3,
|
||||||
|
started_at = CASE WHEN $2::text = 'running' THEN now() ELSE started_at END,
|
||||||
|
finished_at = CASE WHEN $2::text IN ('succeeded','failed') THEN now() ELSE finished_at END
|
||||||
|
WHERE id = $1`,
|
||||||
|
id, status, errMsg,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return mapError(err)
|
||||||
|
}
|
||||||
|
if tag.RowsAffected() == 0 {
|
||||||
|
return domain.ErrNotFound
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetNodeStarted marks a run's node as running, storing its fingerprint and
|
||||||
|
// resolved inputs, and clears any previous error. Returns domain.ErrNotFound
|
||||||
|
// if no such node exists.
|
||||||
|
func (r *LandscapeRepository) SetNodeStarted(ctx context.Context, runID uuid.UUID, code string, fingerprint string, inputs []domain.LPDatasetRef) error {
|
||||||
|
tag, err := r.pool.Exec(ctx,
|
||||||
|
`UPDATE lp_run_nodes
|
||||||
|
SET status = 'running', fingerprint = $3, input_datasets = $4, started_at = now(), error = NULL
|
||||||
|
WHERE run_id = $1 AND node_code = $2`,
|
||||||
|
runID, code, fingerprint, inputs,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return mapError(err)
|
||||||
|
}
|
||||||
|
if tag.RowsAffected() == 0 {
|
||||||
|
return domain.ErrNotFound
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetNodeFinished marks a run's node as finished with the given status,
|
||||||
|
// storing its outputs and error. Returns domain.ErrNotFound if no such node
|
||||||
|
// exists.
|
||||||
|
func (r *LandscapeRepository) SetNodeFinished(ctx context.Context, runID uuid.UUID, code string, status domain.LPNodeStatus, outputs []domain.LPDatasetRef, errMsg *string) error {
|
||||||
|
tag, err := r.pool.Exec(ctx,
|
||||||
|
`UPDATE lp_run_nodes
|
||||||
|
SET status = $3, output_datasets = $4, error = $5, finished_at = now()
|
||||||
|
WHERE run_id = $1 AND node_code = $2`,
|
||||||
|
runID, code, status, outputs, errMsg,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return mapError(err)
|
||||||
|
}
|
||||||
|
if tag.RowsAffected() == 0 {
|
||||||
|
return domain.ErrNotFound
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindReusableOutputs returns the outputs of the most recently succeeded node
|
||||||
|
// with the same code and fingerprint, excluding excludeRun, so its outputs
|
||||||
|
// may be reused instead of recomputed. Returns domain.ErrNotFound if none.
|
||||||
|
func (r *LandscapeRepository) FindReusableOutputs(ctx context.Context, code, fingerprint string, excludeRun uuid.UUID) ([]domain.LPDatasetRef, error) {
|
||||||
|
var outputs []domain.LPDatasetRef
|
||||||
|
err := r.pool.QueryRow(ctx,
|
||||||
|
`SELECT output_datasets FROM lp_run_nodes
|
||||||
|
WHERE node_code = $1 AND fingerprint = $2 AND status = 'succeeded' AND run_id <> $3 AND output_datasets IS NOT NULL
|
||||||
|
ORDER BY finished_at DESC LIMIT 1`,
|
||||||
|
code, fingerprint, excludeRun,
|
||||||
|
).Scan(&outputs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapError(err)
|
||||||
|
}
|
||||||
|
return outputs, nil
|
||||||
|
}
|
||||||
151
internal/repository/postgres/landscape_db_test.go
Normal file
151
internal/repository/postgres/landscape_db_test.go
Normal file
@@ -0,0 +1,151 @@
|
|||||||
|
package postgres
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gis/internal/domain"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
)
|
||||||
|
|
||||||
|
// testPool connects to the database named by TEST_DB_URL (migrations already
|
||||||
|
// applied) or skips the test. Repository SQL is otherwise only exercised by
|
||||||
|
// hand, so these tests are the guard against Postgres-only failures such as
|
||||||
|
// parameter type deduction (SQLSTATE 42P08) that no stub can reproduce.
|
||||||
|
func testPool(t *testing.T) *pgxpool.Pool {
|
||||||
|
t.Helper()
|
||||||
|
url := os.Getenv("TEST_DB_URL")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("TEST_DB_URL not set; skipping database test")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
pool, err := Connect(ctx, url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("connect: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(pool.Close)
|
||||||
|
return pool
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLandscapeRepository_SetRunStatus_AllTransitions(t *testing.T) {
|
||||||
|
pool := testPool(t)
|
||||||
|
repo := NewLandscapeRepository(pool)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
run, err := repo.CreateRun(ctx, domain.LPRun{
|
||||||
|
Status: domain.LPRunQueued,
|
||||||
|
Params: []byte(`{}`),
|
||||||
|
Nodes: []domain.LPRunNode{{NodeCode: "00", Stage: 0, Status: domain.LPNodePending}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CreateRun: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM lp_runs WHERE id = $1`, run.ID) })
|
||||||
|
|
||||||
|
if err := repo.SetRunStatus(ctx, run.ID, domain.LPRunRunning, nil); err != nil {
|
||||||
|
t.Fatalf("SetRunStatus(running): %v", err)
|
||||||
|
}
|
||||||
|
got, err := repo.GetRun(ctx, run.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetRun: %v", err)
|
||||||
|
}
|
||||||
|
if got.Status != domain.LPRunRunning || got.StartedAt == nil || got.FinishedAt != nil {
|
||||||
|
t.Fatalf("after running: status=%s started=%v finished=%v", got.Status, got.StartedAt, got.FinishedAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := "boom"
|
||||||
|
if err := repo.SetRunStatus(ctx, run.ID, domain.LPRunFailed, &msg); err != nil {
|
||||||
|
t.Fatalf("SetRunStatus(failed): %v", err)
|
||||||
|
}
|
||||||
|
got, err = repo.GetRun(ctx, run.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetRun: %v", err)
|
||||||
|
}
|
||||||
|
if got.Status != domain.LPRunFailed || got.FinishedAt == nil || got.Error == nil || *got.Error != msg {
|
||||||
|
t.Fatalf("after failed: status=%s finished=%v error=%v", got.Status, got.FinishedAt, got.Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := repo.SetRunStatus(ctx, run.ID, domain.LPRunSucceeded, nil); err != nil {
|
||||||
|
t.Fatalf("SetRunStatus(succeeded): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLandscapeRepository_NodeLifecycleAndReuse(t *testing.T) {
|
||||||
|
pool := testPool(t)
|
||||||
|
repo := NewLandscapeRepository(pool)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
newRun := func() domain.LPRun {
|
||||||
|
run, err := repo.CreateRun(ctx, domain.LPRun{
|
||||||
|
Status: domain.LPRunQueued,
|
||||||
|
Params: []byte(`{"window_size":2048}`),
|
||||||
|
Overrides: map[string]uuid.UUID{"lp-e": uuid.New()},
|
||||||
|
Nodes: []domain.LPRunNode{{NodeCode: "00", Stage: 0, Status: domain.LPNodePending}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CreateRun: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM lp_runs WHERE id = $1`, run.ID) })
|
||||||
|
return run
|
||||||
|
}
|
||||||
|
|
||||||
|
inputs := []domain.LPDatasetRef{{Group: "lp-z-wr", DatasetID: uuid.New()}}
|
||||||
|
outputs := []domain.LPDatasetRef{{Group: "lp-z-integrated", DatasetID: uuid.New()}}
|
||||||
|
|
||||||
|
first := newRun()
|
||||||
|
if err := repo.SetNodeStarted(ctx, first.ID, "00", "fp-1", inputs); err != nil {
|
||||||
|
t.Fatalf("SetNodeStarted: %v", err)
|
||||||
|
}
|
||||||
|
if err := repo.SetNodeFinished(ctx, first.ID, "00", domain.LPNodeSucceeded, outputs, nil); err != nil {
|
||||||
|
t.Fatalf("SetNodeFinished: %v", err)
|
||||||
|
}
|
||||||
|
got, err := repo.GetRun(ctx, first.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetRun: %v", err)
|
||||||
|
}
|
||||||
|
if len(got.Nodes) != 1 || got.Nodes[0].Status != domain.LPNodeSucceeded ||
|
||||||
|
len(got.Nodes[0].Inputs) != 1 || got.Nodes[0].Inputs[0] != inputs[0] ||
|
||||||
|
len(got.Nodes[0].Outputs) != 1 || got.Nodes[0].Outputs[0] != outputs[0] ||
|
||||||
|
got.Nodes[0].StartedAt == nil || got.Nodes[0].FinishedAt == nil {
|
||||||
|
t.Fatalf("node row not round-tripped: %+v", got.Nodes)
|
||||||
|
}
|
||||||
|
if len(got.Overrides) != 1 {
|
||||||
|
t.Fatalf("overrides not round-tripped: %+v", got.Overrides)
|
||||||
|
}
|
||||||
|
|
||||||
|
second := newRun()
|
||||||
|
reused, err := repo.FindReusableOutputs(ctx, "00", "fp-1", second.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FindReusableOutputs: %v", err)
|
||||||
|
}
|
||||||
|
if len(reused) != 1 || reused[0] != outputs[0] {
|
||||||
|
t.Fatalf("reused = %+v, want %+v", reused, outputs)
|
||||||
|
}
|
||||||
|
if _, err := repo.FindReusableOutputs(ctx, "00", "fp-other", second.ID); !errors.Is(err, domain.ErrNotFound) {
|
||||||
|
t.Fatalf("unknown fingerprint: err = %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
if _, err := repo.FindReusableOutputs(ctx, "00", "fp-1", first.ID); !errors.Is(err, domain.ErrNotFound) {
|
||||||
|
t.Fatalf("own run must be excluded: err = %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
failMsg := "grid mismatch"
|
||||||
|
if err := repo.SetNodeFinished(ctx, second.ID, "00", domain.LPNodeFailed, nil, &failMsg); err != nil {
|
||||||
|
t.Fatalf("SetNodeFinished(failed): %v", err)
|
||||||
|
}
|
||||||
|
runs, err := repo.ListRuns(ctx, 10, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListRuns: %v", err)
|
||||||
|
}
|
||||||
|
if len(runs) < 2 || runs[0].ID != second.ID {
|
||||||
|
t.Fatalf("ListRuns must be newest first: %+v", runs)
|
||||||
|
}
|
||||||
|
if n, err := repo.CountRuns(ctx); err != nil || n < 2 {
|
||||||
|
t.Fatalf("CountRuns = %d, %v", n, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
189
internal/service/dataset_group.go
Normal file
189
internal/service/dataset_group.go
Normal file
@@ -0,0 +1,189 @@
|
|||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gis/internal/domain"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DatasetGroupRepository is the persistence behaviour DatasetGroupService needs.
|
||||||
|
type DatasetGroupRepository interface {
|
||||||
|
Create(ctx context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error)
|
||||||
|
GetByID(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, error)
|
||||||
|
GetByCode(ctx context.Context, code string) (domain.DatasetGroup, error)
|
||||||
|
List(ctx context.Context, code *string, limit, offset int) ([]domain.DatasetGroup, error)
|
||||||
|
Count(ctx context.Context, code *string) (int, error)
|
||||||
|
Update(ctx context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error)
|
||||||
|
Delete(ctx context.Context, id uuid.UUID) error
|
||||||
|
SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.DatasetGroup, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DatasetGroupInput carries the mutable fields of a group.
|
||||||
|
type DatasetGroupInput struct {
|
||||||
|
Code string
|
||||||
|
Name string
|
||||||
|
Description string
|
||||||
|
}
|
||||||
|
|
||||||
|
// DatasetGroupPage is a page of groups.
|
||||||
|
type DatasetGroupPage struct {
|
||||||
|
Items []domain.DatasetGroup
|
||||||
|
Page int
|
||||||
|
PageSize int
|
||||||
|
Total int
|
||||||
|
}
|
||||||
|
|
||||||
|
// DatasetGroupService implements dataset-group business rules.
|
||||||
|
type DatasetGroupService struct {
|
||||||
|
repo DatasetGroupRepository
|
||||||
|
datasets DatasetRepository
|
||||||
|
store ObjectStore
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDatasetGroupService wires the group repository, the dataset repository
|
||||||
|
// (for the "latest" lookup) and the object store (for the group SLD).
|
||||||
|
func NewDatasetGroupService(repo DatasetGroupRepository, datasets DatasetRepository, store ObjectStore) *DatasetGroupService {
|
||||||
|
return &DatasetGroupService{repo: repo, datasets: datasets, store: store}
|
||||||
|
}
|
||||||
|
|
||||||
|
// errInvalidGroupCode is the validation error returned for a non-slug code.
|
||||||
|
func errInvalidGroupCode() error {
|
||||||
|
return fmt.Errorf("%w: code must be a slug (lowercase latin letters, digits, and dashes)", domain.ErrValidation)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create persists a new group after validating its code.
|
||||||
|
func (s *DatasetGroupService) Create(ctx context.Context, in DatasetGroupInput) (domain.DatasetGroup, error) {
|
||||||
|
if !domain.ValidSlug(in.Code) {
|
||||||
|
return domain.DatasetGroup{}, errInvalidGroupCode()
|
||||||
|
}
|
||||||
|
return s.repo.Create(ctx, domain.DatasetGroup{Code: in.Code, Name: in.Name, Description: in.Description})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns one group by id.
|
||||||
|
func (s *DatasetGroupService) Get(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, error) {
|
||||||
|
return s.repo.GetByID(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// List returns a page of groups; page/pageSize are clamped like datasets.
|
||||||
|
func (s *DatasetGroupService) List(ctx context.Context, code *string, page, pageSize int) (DatasetGroupPage, error) {
|
||||||
|
if page < 1 {
|
||||||
|
page = 1
|
||||||
|
}
|
||||||
|
if pageSize < 1 {
|
||||||
|
pageSize = DefaultPageSize
|
||||||
|
}
|
||||||
|
if pageSize > MaxPageSize {
|
||||||
|
pageSize = MaxPageSize
|
||||||
|
}
|
||||||
|
items, err := s.repo.List(ctx, code, pageSize, (page-1)*pageSize)
|
||||||
|
if err != nil {
|
||||||
|
return DatasetGroupPage{}, err
|
||||||
|
}
|
||||||
|
total, err := s.repo.Count(ctx, code)
|
||||||
|
if err != nil {
|
||||||
|
return DatasetGroupPage{}, err
|
||||||
|
}
|
||||||
|
return DatasetGroupPage{Items: items, Page: page, PageSize: pageSize, Total: total}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update replaces a group's editable fields.
|
||||||
|
func (s *DatasetGroupService) Update(ctx context.Context, id uuid.UUID, in DatasetGroupInput) (domain.DatasetGroup, error) {
|
||||||
|
if !domain.ValidSlug(in.Code) {
|
||||||
|
return domain.DatasetGroup{}, errInvalidGroupCode()
|
||||||
|
}
|
||||||
|
return s.repo.Update(ctx, domain.DatasetGroup{ID: id, Code: in.Code, Name: in.Name, Description: in.Description})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes a group (409 via ErrConflict while datasets reference it) and
|
||||||
|
// then best-effort removes its SLD object.
|
||||||
|
func (s *DatasetGroupService) Delete(ctx context.Context, id uuid.UUID) error {
|
||||||
|
g, err := s.repo.GetByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.repo.Delete(ctx, id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if g.SldStorageKey != nil {
|
||||||
|
_ = s.store.Remove(ctx, *g.SldStorageKey) // best-effort: the row is gone
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Latest returns the freshest dataset of the group (greatest end_date, NULLs
|
||||||
|
// last, then newest), optionally restricted by status and file type.
|
||||||
|
func (s *DatasetGroupService) Latest(ctx context.Context, id uuid.UUID, status *string, fileType *domain.FileType) (domain.Dataset, error) {
|
||||||
|
if _, err := s.repo.GetByID(ctx, id); err != nil {
|
||||||
|
return domain.Dataset{}, err
|
||||||
|
}
|
||||||
|
return s.datasets.LatestInGroup(ctx, id, status, fileType)
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadSLD validates and stores the group's shared SLD, replacing any previous one.
|
||||||
|
func (s *DatasetGroupService) UploadSLD(ctx context.Context, id uuid.UUID, in UploadSLDInput) (domain.DatasetGroup, error) {
|
||||||
|
g, err := s.repo.GetByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return domain.DatasetGroup{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
ext := strings.ToLower(filepath.Ext(in.Filename))
|
||||||
|
if !domain.ExtensionAllowedForSLD(ext) {
|
||||||
|
return domain.DatasetGroup{}, fmt.Errorf("%w: sld file must have one of these extensions: %s",
|
||||||
|
domain.ErrValidation, strings.Join(domain.AllowedSLDExtensions, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
head := make([]byte, 512)
|
||||||
|
n, err := io.ReadFull(in.Reader, head)
|
||||||
|
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||||
|
return domain.DatasetGroup{}, fmt.Errorf("read sld upload: %w", err)
|
||||||
|
}
|
||||||
|
head = head[:n]
|
||||||
|
if err := domain.ValidateFileContent(ext, head); err != nil {
|
||||||
|
return domain.DatasetGroup{}, fmt.Errorf("%w: %s", domain.ErrValidation, err)
|
||||||
|
}
|
||||||
|
content := io.MultiReader(bytes.NewReader(head), in.Reader)
|
||||||
|
|
||||||
|
name, err := safeObjectName(in.Filename)
|
||||||
|
if err != nil {
|
||||||
|
return domain.DatasetGroup{}, err
|
||||||
|
}
|
||||||
|
key := path.Join("groups", g.ID.String(), "sld", name)
|
||||||
|
if err := s.store.Put(ctx, key, content, in.Size, in.ContentType); err != nil {
|
||||||
|
return domain.DatasetGroup{}, err
|
||||||
|
}
|
||||||
|
updated, err := s.repo.SetSLD(ctx, id, key, in.Filename, in.ContentType)
|
||||||
|
if err != nil {
|
||||||
|
// Compensate: the row was not updated, so the object would be orphaned.
|
||||||
|
_ = s.store.Remove(ctx, key)
|
||||||
|
return domain.DatasetGroup{}, err
|
||||||
|
}
|
||||||
|
if g.SldStorageKey != nil && *g.SldStorageKey != key {
|
||||||
|
_ = s.store.Remove(ctx, *g.SldStorageKey) // best-effort: drop the replaced file
|
||||||
|
}
|
||||||
|
return updated, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadSLD returns the group and a reader for its SLD; ErrSLDNotFound when
|
||||||
|
// none has been uploaded. The caller must close the reader.
|
||||||
|
func (s *DatasetGroupService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, io.ReadCloser, error) {
|
||||||
|
g, err := s.repo.GetByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return domain.DatasetGroup{}, nil, err
|
||||||
|
}
|
||||||
|
if g.SldStorageKey == nil {
|
||||||
|
return domain.DatasetGroup{}, nil, ErrSLDNotFound
|
||||||
|
}
|
||||||
|
obj, err := s.store.Get(ctx, *g.SldStorageKey)
|
||||||
|
if err != nil {
|
||||||
|
return domain.DatasetGroup{}, nil, err
|
||||||
|
}
|
||||||
|
return g, obj, nil
|
||||||
|
}
|
||||||
283
internal/service/dataset_group_test.go
Normal file
283
internal/service/dataset_group_test.go
Normal file
@@ -0,0 +1,283 @@
|
|||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gis/internal/domain"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// stubGroupRepo is an in-memory DatasetGroupRepository for tests. It also
|
||||||
|
// satisfies the dataset service's groupReader dependency.
|
||||||
|
type stubGroupRepo struct {
|
||||||
|
store map[uuid.UUID]domain.DatasetGroup
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStubGroupRepo() *stubGroupRepo {
|
||||||
|
return &stubGroupRepo{store: map[uuid.UUID]domain.DatasetGroup{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *stubGroupRepo) Create(_ context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) {
|
||||||
|
for _, e := range r.store {
|
||||||
|
if e.Code == g.Code {
|
||||||
|
return domain.DatasetGroup{}, domain.ErrConflict
|
||||||
|
}
|
||||||
|
}
|
||||||
|
g.ID = uuid.New()
|
||||||
|
r.store[g.ID] = g
|
||||||
|
return g, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *stubGroupRepo) GetByID(_ context.Context, id uuid.UUID) (domain.DatasetGroup, error) {
|
||||||
|
g, ok := r.store[id]
|
||||||
|
if !ok {
|
||||||
|
return domain.DatasetGroup{}, domain.ErrNotFound
|
||||||
|
}
|
||||||
|
return g, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *stubGroupRepo) GetByCode(_ context.Context, code string) (domain.DatasetGroup, error) {
|
||||||
|
for _, g := range r.store {
|
||||||
|
if g.Code == code {
|
||||||
|
return g, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return domain.DatasetGroup{}, domain.ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *stubGroupRepo) List(_ context.Context, code *string, _, _ int) ([]domain.DatasetGroup, error) {
|
||||||
|
out := []domain.DatasetGroup{}
|
||||||
|
for _, g := range r.store {
|
||||||
|
if code == nil || g.Code == *code {
|
||||||
|
out = append(out, g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *stubGroupRepo) Count(_ context.Context, code *string) (int, error) {
|
||||||
|
l, _ := r.List(context.Background(), code, 0, 0)
|
||||||
|
return len(l), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *stubGroupRepo) Update(_ context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) {
|
||||||
|
if _, ok := r.store[g.ID]; !ok {
|
||||||
|
return domain.DatasetGroup{}, domain.ErrNotFound
|
||||||
|
}
|
||||||
|
r.store[g.ID] = g
|
||||||
|
return g, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *stubGroupRepo) Delete(_ context.Context, id uuid.UUID) error {
|
||||||
|
if _, ok := r.store[id]; !ok {
|
||||||
|
return domain.ErrNotFound
|
||||||
|
}
|
||||||
|
delete(r.store, id)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *stubGroupRepo) SetSLD(_ context.Context, id uuid.UUID, key, filename, contentType string) (domain.DatasetGroup, error) {
|
||||||
|
g, ok := r.store[id]
|
||||||
|
if !ok {
|
||||||
|
return domain.DatasetGroup{}, domain.ErrNotFound
|
||||||
|
}
|
||||||
|
g.SldStorageKey, g.SldFilename, g.SldContentType = &key, &filename, &contentType
|
||||||
|
r.store[id] = g
|
||||||
|
return g, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_Create_RequiresSlug(t *testing.T) {
|
||||||
|
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore())
|
||||||
|
_, err := svc.Create(context.Background(), DatasetGroupInput{Code: "Bad Code", Name: "x"})
|
||||||
|
if !errors.Is(err, domain.ErrValidation) {
|
||||||
|
t.Fatalf("want ErrValidation, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_Latest_PrefersGreatestEndDateThenCreatedAt(t *testing.T) {
|
||||||
|
groups := newStubGroupRepo()
|
||||||
|
datasets := newStubDatasetRepo()
|
||||||
|
svc := NewDatasetGroupService(groups, datasets, newStubStore())
|
||||||
|
g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s := func(v string) *string { return &v }
|
||||||
|
base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
old := domain.Dataset{ID: uuid.New(), DatasetGroupID: &g.ID, EndDate: s("2019-12-31"), Status: domain.DatasetStatusReady, CreatedAt: base}
|
||||||
|
fresh := domain.Dataset{ID: uuid.New(), DatasetGroupID: &g.ID, EndDate: s("2020-12-31"), Status: domain.DatasetStatusReady, CreatedAt: base.Add(time.Hour)}
|
||||||
|
undated := domain.Dataset{ID: uuid.New(), DatasetGroupID: &g.ID, Status: domain.DatasetStatusReady, CreatedAt: base.Add(2 * time.Hour)}
|
||||||
|
for _, d := range []domain.Dataset{old, fresh, undated} {
|
||||||
|
datasets.store[d.ID] = d
|
||||||
|
}
|
||||||
|
got, err := svc.Latest(context.Background(), g.ID, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.ID != fresh.ID {
|
||||||
|
t.Fatalf("want %s, got %s", fresh.ID, got.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_Latest_NotFoundWhenEmpty(t *testing.T) {
|
||||||
|
groups := newStubGroupRepo()
|
||||||
|
svc := NewDatasetGroupService(groups, newStubDatasetRepo(), newStubStore())
|
||||||
|
g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := svc.Latest(context.Background(), g.ID, nil, nil); !errors.Is(err, domain.ErrNotFound) {
|
||||||
|
t.Fatalf("want ErrNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_Latest_UnknownGroupIsNotFound(t *testing.T) {
|
||||||
|
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore())
|
||||||
|
if _, err := svc.Latest(context.Background(), uuid.New(), nil, nil); !errors.Is(err, domain.ErrNotFound) {
|
||||||
|
t.Fatalf("want ErrNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_UploadSLD_RejectsBadExtension(t *testing.T) {
|
||||||
|
groups := newStubGroupRepo()
|
||||||
|
store := newStubStore()
|
||||||
|
svc := NewDatasetGroupService(groups, newStubDatasetRepo(), store)
|
||||||
|
g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = svc.UploadSLD(context.Background(), g.ID, UploadSLDInput{Filename: "style.txt", ContentType: "text/plain", Size: 3, Reader: strings.NewReader("<x>")})
|
||||||
|
if !errors.Is(err, domain.ErrValidation) {
|
||||||
|
t.Fatalf("want ErrValidation, got %v", err)
|
||||||
|
}
|
||||||
|
if len(store.put) != 0 {
|
||||||
|
t.Fatalf("nothing should be stored on validation failure, got %v", store.put)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_UploadSLD_StoresAndReplaces(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
groups := newStubGroupRepo()
|
||||||
|
store := newStubStore()
|
||||||
|
svc := NewDatasetGroupService(groups, newStubDatasetRepo(), store)
|
||||||
|
g, err := svc.Create(ctx, DatasetGroupInput{Code: "pop", Name: "Population"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
updated, err := svc.UploadSLD(ctx, g.ID, UploadSLDInput{
|
||||||
|
Filename: "style.sld", ContentType: "application/xml", Size: 3, Reader: strings.NewReader("<x>"),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantKey := "groups/" + g.ID.String() + "/sld/style.sld"
|
||||||
|
if updated.SldStorageKey == nil || *updated.SldStorageKey != wantKey {
|
||||||
|
t.Fatalf("want key %s, got %v", wantKey, updated.SldStorageKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second upload under a different name replaces the previous object.
|
||||||
|
if _, err := svc.UploadSLD(ctx, g.ID, UploadSLDInput{
|
||||||
|
Filename: "other.sld", ContentType: "application/xml", Size: 3, Reader: strings.NewReader("<y>"),
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(store.removed) != 1 || store.removed[0] != wantKey {
|
||||||
|
t.Fatalf("want previous sld removed, got %v", store.removed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_DownloadSLD_NotFoundWhenAbsent(t *testing.T) {
|
||||||
|
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore())
|
||||||
|
g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := svc.DownloadSLD(context.Background(), g.ID); !errors.Is(err, ErrSLDNotFound) {
|
||||||
|
t.Fatalf("want ErrSLDNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_Delete_RemovesSLDObject(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
store := newStubStore()
|
||||||
|
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), store)
|
||||||
|
g, err := svc.Create(ctx, DatasetGroupInput{Code: "pop", Name: "Population"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := svc.UploadSLD(ctx, g.ID, UploadSLDInput{
|
||||||
|
Filename: "style.sld", ContentType: "application/xml", Size: 3, Reader: strings.NewReader("<x>"),
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := svc.Delete(ctx, g.ID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantKey := "groups/" + g.ID.String() + "/sld/style.sld"
|
||||||
|
if len(store.removed) != 1 || store.removed[0] != wantKey {
|
||||||
|
t.Fatalf("want sld object removed, got %v", store.removed)
|
||||||
|
}
|
||||||
|
if _, err := svc.Get(ctx, g.ID); !errors.Is(err, domain.ErrNotFound) {
|
||||||
|
t.Fatalf("want ErrNotFound after delete, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_List_ClampsPageSize(t *testing.T) {
|
||||||
|
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore())
|
||||||
|
if _, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
page, err := svc.List(context.Background(), nil, 0, 10_000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if page.Page != 1 || page.PageSize != MaxPageSize {
|
||||||
|
t.Fatalf("want page 1 and page size %d, got %d/%d", MaxPageSize, page.Page, page.PageSize)
|
||||||
|
}
|
||||||
|
if page.Total != 1 {
|
||||||
|
t.Fatalf("want total 1, got %d", page.Total)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_Update_RequiresSlug(t *testing.T) {
|
||||||
|
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore())
|
||||||
|
g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := svc.Update(context.Background(), g.ID, DatasetGroupInput{Code: "Bad Code", Name: "x"}); !errors.Is(err, domain.ErrValidation) {
|
||||||
|
t.Fatalf("want ErrValidation, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetGroupService_UploadSLD_RejectsPathTraversal(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
groups := newStubGroupRepo()
|
||||||
|
store := newStubStore()
|
||||||
|
svc := NewDatasetGroupService(groups, newStubDatasetRepo(), store)
|
||||||
|
g, err := svc.Create(ctx, DatasetGroupInput{Code: "pop", Name: "Population"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
updated, err := svc.UploadSLD(ctx, g.ID, UploadSLDInput{
|
||||||
|
Filename: "../../other/sld/evil.sld", ContentType: "application/xml", Size: 3, Reader: strings.NewReader("<x>"),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantPrefix := "groups/" + g.ID.String() + "/sld/"
|
||||||
|
if updated.SldStorageKey == nil || !strings.HasPrefix(*updated.SldStorageKey, wantPrefix) ||
|
||||||
|
!strings.HasSuffix(*updated.SldStorageKey, "evil.sld") {
|
||||||
|
t.Fatalf("sld key escaped the group prefix: %v", updated.SldStorageKey)
|
||||||
|
}
|
||||||
|
if len(store.put) != 1 || !strings.HasPrefix(store.put[0], wantPrefix) {
|
||||||
|
t.Fatalf("object stored outside the group sld prefix: %v", store.put)
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
808
internal/service/landscape.go
Normal file
808
internal/service/landscape.go
Normal file
@@ -0,0 +1,808 @@
|
|||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gis/internal/domain"
|
||||||
|
"gis/internal/landscape"
|
||||||
|
"gis/internal/raster"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ComputeFunc runs one graph node over its input rasters, writing one raster
|
||||||
|
// per output group into outDir and returning the produced paths by group. It
|
||||||
|
// mirrors landscape.Compute so tests can substitute a stub.
|
||||||
|
type ComputeFunc func(ctx context.Context, rio raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params) (map[string]string, error)
|
||||||
|
|
||||||
|
// ComputeWithWorkersFunc is ComputeFunc plus an explicit per-node worker cap.
|
||||||
|
// It mirrors landscape.ComputeWithWorkers.
|
||||||
|
type ComputeWithWorkersFunc func(ctx context.Context, rio raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params, workers int) (map[string]string, error)
|
||||||
|
|
||||||
|
// Default bounds on a run's parallelism, mirroring config.LandscapeConfig.
|
||||||
|
const (
|
||||||
|
// DefaultMaxWorkers is the total window-worker budget of one run.
|
||||||
|
DefaultMaxWorkers = 8
|
||||||
|
// DefaultMaxConcurrentNodes is how many nodes of a stage run at a time.
|
||||||
|
DefaultMaxConcurrentNodes = 2
|
||||||
|
)
|
||||||
|
|
||||||
|
// terminalWriteTimeout bounds a terminal status write made on a context that
|
||||||
|
// has already been cancelled (see terminalCtx).
|
||||||
|
const terminalWriteTimeout = 10 * time.Second
|
||||||
|
|
||||||
|
// LandscapeOptions bounds a run's parallelism, and with it the worker's peak
|
||||||
|
// memory: MaxConcurrentNodes nodes run at a time, and they share the
|
||||||
|
// MaxWorkers window-worker budget. Zero or negative values fall back to the
|
||||||
|
// defaults.
|
||||||
|
type LandscapeOptions struct {
|
||||||
|
MaxWorkers int
|
||||||
|
MaxConcurrentNodes int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o LandscapeOptions) withDefaults() LandscapeOptions {
|
||||||
|
if o.MaxWorkers < 1 {
|
||||||
|
o.MaxWorkers = DefaultMaxWorkers
|
||||||
|
}
|
||||||
|
if o.MaxConcurrentNodes < 1 {
|
||||||
|
o.MaxConcurrentNodes = DefaultMaxConcurrentNodes
|
||||||
|
}
|
||||||
|
return o
|
||||||
|
}
|
||||||
|
|
||||||
|
// workersPerNode splits the total worker budget over the concurrently running
|
||||||
|
// nodes, so nodes x workers stays at MaxWorkers. Always at least 1.
|
||||||
|
func (o LandscapeOptions) workersPerNode() int {
|
||||||
|
return max(1, o.MaxWorkers/o.MaxConcurrentNodes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LandscapeRepository is the persistence behaviour LandscapeService needs.
|
||||||
|
type LandscapeRepository interface {
|
||||||
|
CreateRun(ctx context.Context, run domain.LPRun) (domain.LPRun, error)
|
||||||
|
GetRun(ctx context.Context, id uuid.UUID) (domain.LPRun, error)
|
||||||
|
ListRuns(ctx context.Context, limit, offset int) ([]domain.LPRun, error)
|
||||||
|
CountRuns(ctx context.Context) (int, error)
|
||||||
|
SetRunStatus(ctx context.Context, id uuid.UUID, status domain.LPRunStatus, errMsg *string) error
|
||||||
|
SetNodeStarted(ctx context.Context, runID uuid.UUID, code string, fingerprint string, inputs []domain.LPDatasetRef) error
|
||||||
|
SetNodeFinished(ctx context.Context, runID uuid.UUID, code string, status domain.LPNodeStatus, outputs []domain.LPDatasetRef, errMsg *string) error
|
||||||
|
FindReusableOutputs(ctx context.Context, code, fingerprint string, excludeRun uuid.UUID) ([]domain.LPDatasetRef, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LandscapeJobEnqueuer schedules asynchronous Landscape Program runs.
|
||||||
|
type LandscapeJobEnqueuer interface {
|
||||||
|
EnqueueRun(ctx context.Context, runID uuid.UUID) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// datasetPublisher is the slice of DatasetService the Landscape Program needs:
|
||||||
|
// uploading a slot input and publishing a computed result.
|
||||||
|
type datasetPublisher interface {
|
||||||
|
Upload(ctx context.Context, in UploadInput) (domain.Dataset, error)
|
||||||
|
CreateRasterFromFile(ctx context.Context, in CreateRasterInput) (domain.Dataset, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DatasetService is what the landscape service publishes through in production.
|
||||||
|
var _ datasetPublisher = (*DatasetService)(nil)
|
||||||
|
|
||||||
|
// SlotInfo describes one leaf input slot and the dataset currently filling it.
|
||||||
|
type SlotInfo struct {
|
||||||
|
Code string `json:"code"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
GroupID uuid.UUID `json:"group_id"`
|
||||||
|
Latest *domain.DatasetSummary `json:"latest"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResultInfo describes one computed output group and its freshest ready dataset.
|
||||||
|
type ResultInfo struct {
|
||||||
|
Group string `json:"group"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
GroupID uuid.UUID `json:"group_id"`
|
||||||
|
Latest *domain.DatasetSummary `json:"latest"`
|
||||||
|
CogURL *string `json:"cog_url"`
|
||||||
|
SldURL *string `json:"sld_url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SlotUploadInput carries a raster upload destined for one input slot.
|
||||||
|
type SlotUploadInput struct {
|
||||||
|
Filename string
|
||||||
|
ContentType string
|
||||||
|
Size int64
|
||||||
|
Reader io.Reader
|
||||||
|
Name string
|
||||||
|
StartDate *string
|
||||||
|
EndDate *string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunInput carries the options of a new Landscape Program run.
|
||||||
|
type RunInput struct {
|
||||||
|
// FromStage forces recomputation from this stage on (0..3); nil recomputes
|
||||||
|
// only what changed.
|
||||||
|
FromStage *int
|
||||||
|
// Overrides pins a specific dataset for a group code instead of resolving
|
||||||
|
// the freshest ready one.
|
||||||
|
Overrides map[string]uuid.UUID
|
||||||
|
Params landscape.Params
|
||||||
|
}
|
||||||
|
|
||||||
|
// LPRunPage is a page of runs with pagination metadata.
|
||||||
|
type LPRunPage struct {
|
||||||
|
Items []domain.LPRun
|
||||||
|
Page int
|
||||||
|
PageSize int
|
||||||
|
Total int
|
||||||
|
}
|
||||||
|
|
||||||
|
// LandscapeService plans and executes Landscape Program runs.
|
||||||
|
type LandscapeService struct {
|
||||||
|
runs LandscapeRepository
|
||||||
|
datasets DatasetRepository
|
||||||
|
groups DatasetGroupRepository
|
||||||
|
categories categoryReader
|
||||||
|
publisher datasetPublisher
|
||||||
|
store ObjectStore
|
||||||
|
jobs LandscapeJobEnqueuer
|
||||||
|
rio raster.IO
|
||||||
|
// compute is the legacy worker-count-agnostic entry point, kept so an
|
||||||
|
// existing ComputeFunc still wires up; runNode always calls
|
||||||
|
// computeWithWorkers, which wraps compute when one was provided.
|
||||||
|
compute ComputeFunc
|
||||||
|
computeWithWorkers ComputeWithWorkersFunc
|
||||||
|
opts LandscapeOptions
|
||||||
|
log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLandscapeService wires the run repository, the dataset and group
|
||||||
|
// repositories (input resolution), the category reader (the "landscape"
|
||||||
|
// category every result is filed under), the dataset publisher (result
|
||||||
|
// datasets and slot uploads), the object store (input downloads), the job
|
||||||
|
// enqueuer, and the raster IO / compute function the graph runs on. Pass a nil
|
||||||
|
// compute (the production wiring) to run landscape.ComputeWithWorkers with the
|
||||||
|
// per-node share of opts.MaxWorkers; a non-nil ComputeFunc is honoured for
|
||||||
|
// backward compatibility but never sees the worker cap. opts bounds the run's
|
||||||
|
// parallelism; zero fields fall back to the package defaults.
|
||||||
|
func NewLandscapeService(
|
||||||
|
runs LandscapeRepository,
|
||||||
|
datasets DatasetRepository,
|
||||||
|
groups DatasetGroupRepository,
|
||||||
|
categories categoryReader,
|
||||||
|
publisher datasetPublisher,
|
||||||
|
store ObjectStore,
|
||||||
|
jobs LandscapeJobEnqueuer,
|
||||||
|
rio raster.IO,
|
||||||
|
compute ComputeFunc,
|
||||||
|
log *slog.Logger,
|
||||||
|
opts LandscapeOptions,
|
||||||
|
) *LandscapeService {
|
||||||
|
// Default to the workers-aware entry point; wrap a supplied legacy
|
||||||
|
// ComputeFunc (which has no worker parameter) when one is given.
|
||||||
|
withWorkers := ComputeWithWorkersFunc(landscape.ComputeWithWorkers)
|
||||||
|
if compute != nil {
|
||||||
|
withWorkers = func(ctx context.Context, rio raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params, _ int) (map[string]string, error) {
|
||||||
|
return compute(ctx, rio, node, inputs, outDir, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &LandscapeService{
|
||||||
|
runs: runs,
|
||||||
|
datasets: datasets,
|
||||||
|
groups: groups,
|
||||||
|
categories: categories,
|
||||||
|
publisher: publisher,
|
||||||
|
store: store,
|
||||||
|
jobs: jobs,
|
||||||
|
rio: rio,
|
||||||
|
compute: compute,
|
||||||
|
computeWithWorkers: withWorkers,
|
||||||
|
opts: opts.withDefaults(),
|
||||||
|
log: log,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Slots lists every leaf input slot with its group and latest dataset (any
|
||||||
|
// status), so the UI can show what has been uploaded.
|
||||||
|
func (s *LandscapeService) Slots(ctx context.Context) ([]SlotInfo, error) {
|
||||||
|
out := make([]SlotInfo, 0, len(landscape.Slots))
|
||||||
|
for _, slot := range landscape.Slots {
|
||||||
|
g, err := s.groups.GetByCode(ctx, slot.Code)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("dataset group %q: %w", slot.Code, err)
|
||||||
|
}
|
||||||
|
info := SlotInfo{Code: slot.Code, Title: slot.Title, GroupID: g.ID}
|
||||||
|
latest, err := s.datasets.LatestInGroup(ctx, g.ID, nil, nil)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
summary := summaryOf(latest)
|
||||||
|
info.Latest = &summary
|
||||||
|
case errors.Is(err, domain.ErrNotFound):
|
||||||
|
// An empty slot is expected before the first upload.
|
||||||
|
default:
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, info)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadToSlot stores a raster under the slot's dataset group in the landscape
|
||||||
|
// category, through the regular upload path (COG conversion is enqueued).
|
||||||
|
// Returns domain.ErrNotFound when the code is not a known slot.
|
||||||
|
func (s *LandscapeService) UploadToSlot(ctx context.Context, code string, in SlotUploadInput) (domain.Dataset, error) {
|
||||||
|
slot, ok := landscape.SlotByCode(code)
|
||||||
|
if !ok {
|
||||||
|
return domain.Dataset{}, fmt.Errorf("%w: unknown landscape slot %q", domain.ErrNotFound, code)
|
||||||
|
}
|
||||||
|
g, err := s.groups.GetByCode(ctx, slot.Code)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Dataset{}, fmt.Errorf("dataset group %q: %w", slot.Code, err)
|
||||||
|
}
|
||||||
|
cat, err := s.categories.GetByCode(ctx, landscape.CategoryCode)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Dataset{}, fmt.Errorf("category %q: %w", landscape.CategoryCode, err)
|
||||||
|
}
|
||||||
|
name := in.Name
|
||||||
|
if name == "" {
|
||||||
|
name = in.Filename
|
||||||
|
}
|
||||||
|
return s.publisher.Upload(ctx, UploadInput{
|
||||||
|
CategoryID: cat.ID,
|
||||||
|
DatasetGroupID: &g.ID,
|
||||||
|
Name: name,
|
||||||
|
Filename: in.Filename,
|
||||||
|
FileType: domain.FileTypeRaster,
|
||||||
|
ContentType: in.ContentType,
|
||||||
|
Size: in.Size,
|
||||||
|
Reader: in.Reader,
|
||||||
|
StartDate: in.StartDate,
|
||||||
|
EndDate: in.EndDate,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreateRun validates the params and overrides, persists a queued run with one
|
||||||
|
// pending node per graph node, and publishes the worker job.
|
||||||
|
func (s *LandscapeService) CreateRun(ctx context.Context, in RunInput) (domain.LPRun, error) {
|
||||||
|
params := in.Params.WithDefaults()
|
||||||
|
if err := params.Validate(); err != nil {
|
||||||
|
return domain.LPRun{}, fmt.Errorf("%w: %s", domain.ErrValidation, err)
|
||||||
|
}
|
||||||
|
if in.FromStage != nil {
|
||||||
|
if *in.FromStage < 0 || *in.FromStage > maxStage() {
|
||||||
|
return domain.LPRun{}, fmt.Errorf("%w: from_stage must be in 0..%d", domain.ErrValidation, maxStage())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
overrides := map[string]uuid.UUID{}
|
||||||
|
for group, id := range in.Overrides {
|
||||||
|
if !knownGroup(group) {
|
||||||
|
return domain.LPRun{}, fmt.Errorf("%w: unknown landscape group %q in overrides", domain.ErrValidation, group)
|
||||||
|
}
|
||||||
|
d, err := s.datasets.GetByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, domain.ErrNotFound) {
|
||||||
|
return domain.LPRun{}, fmt.Errorf("%w: override for %q: dataset %s does not exist", domain.ErrValidation, group, id)
|
||||||
|
}
|
||||||
|
return domain.LPRun{}, err
|
||||||
|
}
|
||||||
|
if d.FileType != domain.FileTypeRaster {
|
||||||
|
return domain.LPRun{}, fmt.Errorf("%w: override for %q: dataset %s is not a raster", domain.ErrValidation, group, id)
|
||||||
|
}
|
||||||
|
if d.Status != domain.DatasetStatusReady {
|
||||||
|
return domain.LPRun{}, fmt.Errorf("%w: override for %q: dataset %s is not ready", domain.ErrValidation, group, id)
|
||||||
|
}
|
||||||
|
overrides[group] = id
|
||||||
|
}
|
||||||
|
|
||||||
|
// lp_runs.params is NOT NULL, and the kernels misbehave on a partially
|
||||||
|
// filled Params, so the defaulted-and-validated value is what we persist.
|
||||||
|
paramsJSON, err := json.Marshal(params)
|
||||||
|
if err != nil {
|
||||||
|
return domain.LPRun{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes := make([]domain.LPRunNode, 0, len(landscape.Nodes))
|
||||||
|
for _, n := range landscape.Nodes {
|
||||||
|
nodes = append(nodes, domain.LPRunNode{NodeCode: n.Code, Stage: n.Stage, Status: domain.LPNodePending})
|
||||||
|
}
|
||||||
|
|
||||||
|
run, err := s.runs.CreateRun(ctx, domain.LPRun{
|
||||||
|
Status: domain.LPRunQueued,
|
||||||
|
FromStage: in.FromStage,
|
||||||
|
Params: paramsJSON,
|
||||||
|
Overrides: overrides,
|
||||||
|
Nodes: nodes,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return domain.LPRun{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.jobs.EnqueueRun(ctx, run.ID); err != nil {
|
||||||
|
msg := fmt.Sprintf("failed to enqueue: %s", err)
|
||||||
|
if setErr := s.runs.SetRunStatus(ctx, run.ID, domain.LPRunFailed, &msg); setErr != nil {
|
||||||
|
s.log.Error("mark landscape run failed", "run_id", run.ID, "error", setErr)
|
||||||
|
}
|
||||||
|
return domain.LPRun{}, fmt.Errorf("enqueue landscape run: %w", err)
|
||||||
|
}
|
||||||
|
return run, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetRun returns one run with its nodes.
|
||||||
|
func (s *LandscapeService) GetRun(ctx context.Context, id uuid.UUID) (domain.LPRun, error) {
|
||||||
|
return s.runs.GetRun(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListRuns returns a page of runs, newest first.
|
||||||
|
func (s *LandscapeService) ListRuns(ctx context.Context, page, pageSize int) (LPRunPage, error) {
|
||||||
|
if page < 1 {
|
||||||
|
page = 1
|
||||||
|
}
|
||||||
|
if pageSize < 1 {
|
||||||
|
pageSize = DefaultPageSize
|
||||||
|
}
|
||||||
|
if pageSize > MaxPageSize {
|
||||||
|
pageSize = MaxPageSize
|
||||||
|
}
|
||||||
|
items, err := s.runs.ListRuns(ctx, pageSize, (page-1)*pageSize)
|
||||||
|
if err != nil {
|
||||||
|
return LPRunPage{}, err
|
||||||
|
}
|
||||||
|
total, err := s.runs.CountRuns(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return LPRunPage{}, err
|
||||||
|
}
|
||||||
|
return LPRunPage{Items: items, Page: page, PageSize: pageSize, Total: total}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Results lists every output group with its freshest ready dataset and the URLs
|
||||||
|
// to render it.
|
||||||
|
func (s *LandscapeService) Results(ctx context.Context) ([]ResultInfo, error) {
|
||||||
|
ready := domain.DatasetStatusReady
|
||||||
|
fileType := domain.FileTypeRaster
|
||||||
|
|
||||||
|
out := make([]ResultInfo, 0, len(landscape.Outputs))
|
||||||
|
for _, o := range landscape.Outputs {
|
||||||
|
g, err := s.groups.GetByCode(ctx, o.Code)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("dataset group %q: %w", o.Code, err)
|
||||||
|
}
|
||||||
|
info := ResultInfo{Group: o.Code, Title: o.Title, GroupID: g.ID}
|
||||||
|
if node, ok := landscape.NodeForOutput(o.Code); ok {
|
||||||
|
info.Node = node.Code
|
||||||
|
}
|
||||||
|
|
||||||
|
latest, err := s.datasets.LatestInGroup(ctx, g.ID, &ready, &fileType)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
summary := summaryOf(latest)
|
||||||
|
info.Latest = &summary
|
||||||
|
cog := "/datasets/" + latest.ID.String() + ".cog"
|
||||||
|
info.CogURL = &cog
|
||||||
|
if g.SldStorageKey != nil {
|
||||||
|
sld := "/dataset-groups/" + g.ID.String() + "/sld"
|
||||||
|
info.SldURL = &sld
|
||||||
|
}
|
||||||
|
case errors.Is(err, domain.ErrNotFound):
|
||||||
|
// Nothing computed for this group yet.
|
||||||
|
default:
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, info)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarkRunFailed records a terminal failure on a run so it does not stay stuck
|
||||||
|
// in an in-progress status.
|
||||||
|
func (s *LandscapeService) MarkRunFailed(ctx context.Context, runID uuid.UUID, reason string) error {
|
||||||
|
wctx, cancel := terminalCtx(ctx)
|
||||||
|
defer cancel()
|
||||||
|
return s.runs.SetRunStatus(wctx, runID, domain.LPRunFailed, &reason)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute runs the whole graph for one run: stages in order, nodes within a
|
||||||
|
// stage concurrently. The first failing node fails the run and later stages are
|
||||||
|
// not started. It is the worker entry point.
|
||||||
|
func (s *LandscapeService) Execute(ctx context.Context, runID uuid.UUID) error {
|
||||||
|
run, err := s.runs.GetRun(ctx, runID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// A redelivered or stale message must not rerun a finished run. "running"
|
||||||
|
// is allowed through so a run whose worker crashed mid-flight still
|
||||||
|
// recovers on redelivery.
|
||||||
|
if run.Status != domain.LPRunQueued && run.Status != domain.LPRunRunning {
|
||||||
|
s.log.Info("landscape run already terminal, ignoring redelivery", "run_id", runID, "status", run.Status)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var params landscape.Params
|
||||||
|
if len(run.Params) > 0 {
|
||||||
|
if err := json.Unmarshal(run.Params, ¶ms); err != nil {
|
||||||
|
return s.failRun(ctx, runID, fmt.Errorf("decode run params: %w", err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
params = params.WithDefaults()
|
||||||
|
|
||||||
|
cat, err := s.categories.GetByCode(ctx, landscape.CategoryCode)
|
||||||
|
if err != nil {
|
||||||
|
return s.failRun(ctx, runID, fmt.Errorf("category %q: %w", landscape.CategoryCode, err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.runs.SetRunStatus(ctx, runID, domain.LPRunRunning, nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Re-read so started_at is available for result dataset names; on failure
|
||||||
|
// keep the pre-transition copy and fall back to time.Now() for the names.
|
||||||
|
if started, err := s.runs.GetRun(ctx, runID); err == nil {
|
||||||
|
run = started
|
||||||
|
} else {
|
||||||
|
s.log.Warn("re-read landscape run after marking it running", "run_id", runID, "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
state := &runState{produced: map[string]uuid.UUID{}}
|
||||||
|
for _, stage := range landscape.Stages() {
|
||||||
|
// WithContext so the first failing node cancels its siblings: Compute
|
||||||
|
// honours cancellation, so a doomed stage stops reading rasters
|
||||||
|
// instead of running to completion. Terminal status writes use
|
||||||
|
// terminalCtx, so they still land on the cancelled context.
|
||||||
|
g, gctx := errgroup.WithContext(ctx)
|
||||||
|
// Each node's Compute spins its own worker pool (each worker holding a
|
||||||
|
// window buffer per input band), so peak memory is the product of the
|
||||||
|
// two limits. Bound the node fan-out here and split the worker budget
|
||||||
|
// over it so that product stays at opts.MaxWorkers.
|
||||||
|
g.SetLimit(s.opts.MaxConcurrentNodes)
|
||||||
|
for _, node := range stage {
|
||||||
|
g.Go(func() error { return s.runNode(gctx, run, node, params, cat, state) })
|
||||||
|
}
|
||||||
|
// Wait for every node of the stage: each one records its own outcome,
|
||||||
|
// and a sibling's failure must not leave a node half-recorded.
|
||||||
|
if err := g.Wait(); err != nil {
|
||||||
|
return s.failRun(ctx, runID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return s.runs.SetRunStatus(ctx, runID, domain.LPRunSucceeded, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// terminalCtx derives a bounded context that outlives the cancellation of ctx.
|
||||||
|
// Terminal status writes must happen even when the run failed *because* the
|
||||||
|
// context was cancelled (shutdown, a sibling node failing the stage); on the
|
||||||
|
// original context they would be refused and leave the run stuck in "running".
|
||||||
|
func terminalCtx(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||||
|
return context.WithTimeout(context.WithoutCancel(ctx), terminalWriteTimeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
// failRun marks the run failed with cause's message and returns cause.
|
||||||
|
func (s *LandscapeService) failRun(ctx context.Context, runID uuid.UUID, cause error) error {
|
||||||
|
msg := cause.Error()
|
||||||
|
wctx, cancel := terminalCtx(ctx)
|
||||||
|
defer cancel()
|
||||||
|
if err := s.runs.SetRunStatus(wctx, runID, domain.LPRunFailed, &msg); err != nil {
|
||||||
|
s.log.Error("mark landscape run failed", "run_id", runID, "error", err)
|
||||||
|
}
|
||||||
|
return cause
|
||||||
|
}
|
||||||
|
|
||||||
|
// runState carries what the run has produced so far, shared by the concurrent
|
||||||
|
// nodes of a stage.
|
||||||
|
type runState struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
produced map[string]uuid.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *runState) get(group string) (uuid.UUID, bool) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
id, ok := r.produced[group]
|
||||||
|
return id, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *runState) record(refs []domain.LPDatasetRef) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
for _, ref := range refs {
|
||||||
|
r.produced[ref.Group] = ref.DatasetID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runNode resolves a node's inputs, reuses a previous result when the
|
||||||
|
// fingerprint matches, and otherwise computes and publishes its outputs.
|
||||||
|
func (s *LandscapeService) runNode(ctx context.Context, run domain.LPRun, node landscape.Node, params landscape.Params, cat domain.Category, state *runState) error {
|
||||||
|
refs, inputs, err := s.resolveInputs(ctx, run, node, state)
|
||||||
|
if err != nil {
|
||||||
|
return s.finishNodeFailed(ctx, run.ID, node, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fp := fingerprint(node, refs, params)
|
||||||
|
if err := s.runs.SetNodeStarted(ctx, run.ID, node.Code, fp, refs); err != nil {
|
||||||
|
// Best-effort, so the node is not left pending while the run is failed.
|
||||||
|
return s.finishNodeFailed(ctx, run.ID, node, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if run.FromStage == nil || node.Stage < *run.FromStage {
|
||||||
|
reused, ok, err := s.reusableOutputs(ctx, run.ID, node, fp)
|
||||||
|
if err != nil {
|
||||||
|
return s.finishNodeFailed(ctx, run.ID, node, err)
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
state.record(reused)
|
||||||
|
return s.runs.SetNodeFinished(ctx, run.ID, node.Code, domain.LPNodeSkipped, reused, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
outputs, err := s.computeNode(ctx, run, node, params, cat, refs, inputs)
|
||||||
|
if err != nil {
|
||||||
|
return s.finishNodeFailed(ctx, run.ID, node, err)
|
||||||
|
}
|
||||||
|
state.record(outputs)
|
||||||
|
if err := s.runs.SetNodeFinished(ctx, run.ID, node.Code, domain.LPNodeSucceeded, outputs, nil); err != nil {
|
||||||
|
// Best-effort, so the node is not left running while the run is failed.
|
||||||
|
return s.finishNodeFailed(ctx, run.ID, node, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// finishNodeFailed records the failure on the node and returns the cause.
|
||||||
|
func (s *LandscapeService) finishNodeFailed(ctx context.Context, runID uuid.UUID, node landscape.Node, cause error) error {
|
||||||
|
msg := cause.Error()
|
||||||
|
wctx, cancel := terminalCtx(ctx)
|
||||||
|
defer cancel()
|
||||||
|
if err := s.runs.SetNodeFinished(wctx, runID, node.Code, domain.LPNodeFailed, nil, &msg); err != nil {
|
||||||
|
s.log.Error("mark landscape node failed", "run_id", runID, "node", node.Code, "error", err)
|
||||||
|
}
|
||||||
|
return cause
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveInputs picks one dataset per input group: an explicit override first,
|
||||||
|
// then whatever this run already produced for the group, then the freshest
|
||||||
|
// ready raster of the group.
|
||||||
|
func (s *LandscapeService) resolveInputs(ctx context.Context, run domain.LPRun, node landscape.Node, state *runState) ([]domain.LPDatasetRef, []domain.Dataset, error) {
|
||||||
|
ready := domain.DatasetStatusReady
|
||||||
|
fileType := domain.FileTypeRaster
|
||||||
|
|
||||||
|
refs := make([]domain.LPDatasetRef, 0, len(node.Inputs))
|
||||||
|
datasets := make([]domain.Dataset, 0, len(node.Inputs))
|
||||||
|
|
||||||
|
for _, group := range node.Inputs {
|
||||||
|
var (
|
||||||
|
d domain.Dataset
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
switch id, ok := s.pinnedInput(run, group, state); {
|
||||||
|
case ok:
|
||||||
|
d, err = s.datasets.GetByID(ctx, id)
|
||||||
|
default:
|
||||||
|
g, gerr := s.groups.GetByCode(ctx, group)
|
||||||
|
if gerr != nil {
|
||||||
|
return nil, nil, fmt.Errorf("node %s: dataset group %q: %w", node.Code, group, gerr)
|
||||||
|
}
|
||||||
|
d, err = s.datasets.LatestInGroup(ctx, g.ID, &ready, &fileType)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, domain.ErrNotFound) {
|
||||||
|
return nil, nil, fmt.Errorf("node %s: no ready raster dataset in group %q", node.Code, group)
|
||||||
|
}
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
refs = append(refs, domain.LPDatasetRef{Group: group, DatasetID: d.ID})
|
||||||
|
datasets = append(datasets, d)
|
||||||
|
}
|
||||||
|
return refs, datasets, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// pinnedInput returns the dataset already decided for a group: the run's
|
||||||
|
// override, or what an earlier node of this run produced.
|
||||||
|
func (s *LandscapeService) pinnedInput(run domain.LPRun, group string, state *runState) (uuid.UUID, bool) {
|
||||||
|
if id, ok := run.Overrides[group]; ok {
|
||||||
|
return id, true
|
||||||
|
}
|
||||||
|
return state.get(group)
|
||||||
|
}
|
||||||
|
|
||||||
|
// reusableOutputs reports whether a previous succeeded node with the same
|
||||||
|
// fingerprint left outputs that all still exist and are ready.
|
||||||
|
func (s *LandscapeService) reusableOutputs(ctx context.Context, runID uuid.UUID, node landscape.Node, fp string) ([]domain.LPDatasetRef, bool, error) {
|
||||||
|
refs, err := s.runs.FindReusableOutputs(ctx, node.Code, fp, runID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, domain.ErrNotFound) {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
if len(refs) != len(node.Outputs) {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
for _, ref := range refs {
|
||||||
|
d, err := s.datasets.GetByID(ctx, ref.DatasetID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, domain.ErrNotFound) {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
if d.Status != domain.DatasetStatusReady {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return refs, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// computeNode downloads the node's inputs into a temp dir, computes it, and
|
||||||
|
// publishes one raster dataset per output group. Temp files are always removed.
|
||||||
|
func (s *LandscapeService) computeNode(
|
||||||
|
ctx context.Context,
|
||||||
|
run domain.LPRun,
|
||||||
|
node landscape.Node,
|
||||||
|
params landscape.Params,
|
||||||
|
cat domain.Category,
|
||||||
|
refs []domain.LPDatasetRef,
|
||||||
|
inputs []domain.Dataset,
|
||||||
|
) ([]domain.LPDatasetRef, error) {
|
||||||
|
// landscape.Compute does not create outDir, so we own the whole directory.
|
||||||
|
dir, err := os.MkdirTemp("", "lp-"+node.Code+"-*")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("node %s: temp dir: %w", node.Code, err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
|
paths := make(map[string]string, len(inputs))
|
||||||
|
for i, d := range inputs {
|
||||||
|
group := refs[i].Group
|
||||||
|
// Downstream nodes read the original object, never the COG, so they
|
||||||
|
// never wait for the conversion job.
|
||||||
|
path := filepath.Join(dir, "in-"+group+".tif")
|
||||||
|
if err := s.download(ctx, d.StorageKey, path); err != nil {
|
||||||
|
return nil, fmt.Errorf("node %s: download %q: %w", node.Code, group, err)
|
||||||
|
}
|
||||||
|
paths[group] = path
|
||||||
|
}
|
||||||
|
|
||||||
|
outPaths, err := s.computeWithWorkers(ctx, s.rio, node, paths, dir, params, s.opts.workersPerNode())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
start, end := periodOf(inputs)
|
||||||
|
date := time.Now()
|
||||||
|
if run.StartedAt != nil {
|
||||||
|
date = *run.StartedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
meta, err := json.Marshal(map[string]any{
|
||||||
|
"lp_run_id": run.ID,
|
||||||
|
"lp_node": node.Code,
|
||||||
|
"inputs": refs,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
outputs := make([]domain.LPDatasetRef, 0, len(node.Outputs))
|
||||||
|
for _, group := range node.Outputs {
|
||||||
|
path, ok := outPaths[group]
|
||||||
|
if !ok || path == "" {
|
||||||
|
return nil, fmt.Errorf("node %s: compute produced no output for group %q", node.Code, group)
|
||||||
|
}
|
||||||
|
g, err := s.groups.GetByCode(ctx, group)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("node %s: dataset group %q: %w", node.Code, group, err)
|
||||||
|
}
|
||||||
|
title := group
|
||||||
|
if o, ok := landscape.OutputByCode(group); ok {
|
||||||
|
title = o.Title
|
||||||
|
}
|
||||||
|
d, err := s.publisher.CreateRasterFromFile(ctx, CreateRasterInput{
|
||||||
|
CategoryID: cat.ID,
|
||||||
|
DatasetGroupID: &g.ID,
|
||||||
|
Name: title + " " + date.Format("2006-01-02"),
|
||||||
|
Filename: group + ".tif",
|
||||||
|
Path: path,
|
||||||
|
StartDate: start,
|
||||||
|
EndDate: end,
|
||||||
|
Meta: meta,
|
||||||
|
Automated: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("node %s: publish %q: %w", node.Code, group, err)
|
||||||
|
}
|
||||||
|
outputs = append(outputs, domain.LPDatasetRef{Group: group, DatasetID: d.ID})
|
||||||
|
}
|
||||||
|
return outputs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// download copies an object from the store to a local path.
|
||||||
|
func (s *LandscapeService) download(ctx context.Context, key, path string) error {
|
||||||
|
obj, err := s.store.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer obj.Close()
|
||||||
|
|
||||||
|
f, err := os.Create(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := io.Copy(f, obj); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return f.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// fingerprint is a stable hash of the node, its resolved inputs and the
|
||||||
|
// node-relevant params.
|
||||||
|
func fingerprint(node landscape.Node, inputs []domain.LPDatasetRef, p landscape.Params) string {
|
||||||
|
sorted := append([]domain.LPDatasetRef(nil), inputs...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Group < sorted[j].Group })
|
||||||
|
payload, _ := json.Marshal(struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Inputs []domain.LPDatasetRef `json:"inputs"`
|
||||||
|
Params landscape.Params `json:"params"`
|
||||||
|
}{node.Code, sorted, p.ForNode(node.Code)})
|
||||||
|
sum := sha256.Sum256(payload)
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// periodOf returns the min start_date and max end_date over the inputs, or nil
|
||||||
|
// when every input leaves the respective bound unset.
|
||||||
|
func periodOf(inputs []domain.Dataset) (*string, *string) {
|
||||||
|
var start, end *string
|
||||||
|
for _, d := range inputs {
|
||||||
|
if d.StartDate != nil && (start == nil || *d.StartDate < *start) {
|
||||||
|
start = d.StartDate
|
||||||
|
}
|
||||||
|
if d.EndDate != nil && (end == nil || *d.EndDate > *end) {
|
||||||
|
end = d.EndDate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return start, end
|
||||||
|
}
|
||||||
|
|
||||||
|
// knownGroup reports whether a group code names an input slot or a node output.
|
||||||
|
func knownGroup(code string) bool {
|
||||||
|
if _, ok := landscape.SlotByCode(code); ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
_, ok := landscape.OutputByCode(code)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxStage returns the highest stage number in the graph.
|
||||||
|
func maxStage() int {
|
||||||
|
highest := 0
|
||||||
|
for _, n := range landscape.Nodes {
|
||||||
|
if n.Stage > highest {
|
||||||
|
highest = n.Stage
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return highest
|
||||||
|
}
|
||||||
|
|
||||||
|
// summaryOf projects a dataset onto the summary shape used in list responses.
|
||||||
|
func summaryOf(d domain.Dataset) domain.DatasetSummary {
|
||||||
|
return domain.DatasetSummary{
|
||||||
|
ID: d.ID,
|
||||||
|
CategoryID: d.CategoryID,
|
||||||
|
DatasetGroupID: d.DatasetGroupID,
|
||||||
|
StartDate: d.StartDate,
|
||||||
|
EndDate: d.EndDate,
|
||||||
|
Name: d.Name,
|
||||||
|
Description: d.Description,
|
||||||
|
Unit: d.Unit,
|
||||||
|
FileType: d.FileType,
|
||||||
|
SizeBytes: d.SizeBytes,
|
||||||
|
Status: d.Status,
|
||||||
|
CreatedAt: d.CreatedAt,
|
||||||
|
UpdatedAt: d.UpdatedAt,
|
||||||
|
}
|
||||||
|
}
|
||||||
1530
internal/service/landscape_test.go
Normal file
1530
internal/service/landscape_test.go
Normal file
File diff suppressed because it is too large
Load Diff
236
internal/transport/http/dataset_group_handler.go
Normal file
236
internal/transport/http/dataset_group_handler.go
Normal file
@@ -0,0 +1,236 @@
|
|||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gis/internal/domain"
|
||||||
|
"gis/internal/service"
|
||||||
|
"gis/pkg/httputil"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
"github.com/go-playground/validator/v10"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DatasetGroupHandler serves the /dataset-groups routes.
|
||||||
|
type DatasetGroupHandler struct {
|
||||||
|
svc *service.DatasetGroupService
|
||||||
|
validate *validator.Validate
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDatasetGroupHandler returns a DatasetGroupHandler.
|
||||||
|
func NewDatasetGroupHandler(svc *service.DatasetGroupService, validate *validator.Validate) *DatasetGroupHandler {
|
||||||
|
return &DatasetGroupHandler{svc: svc, validate: validate}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register mounts the dataset-group routes on r.
|
||||||
|
func (h *DatasetGroupHandler) Register(r chi.Router) {
|
||||||
|
r.Get("/", h.list)
|
||||||
|
r.Post("/", h.create)
|
||||||
|
r.Get("/{id}", h.get)
|
||||||
|
r.Put("/{id}", h.update)
|
||||||
|
r.Delete("/{id}", h.delete)
|
||||||
|
r.Get("/{id}/latest", h.latest)
|
||||||
|
r.Post("/{id}/sld", h.uploadSLD)
|
||||||
|
r.Get("/{id}/sld", h.downloadSLD)
|
||||||
|
}
|
||||||
|
|
||||||
|
type datasetGroupRequest struct {
|
||||||
|
Code string `json:"code" validate:"required,max=255,slug"`
|
||||||
|
Name string `json:"name" validate:"required,max=255"`
|
||||||
|
Description string `json:"description" validate:"max=2000"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r datasetGroupRequest) toInput() service.DatasetGroupInput {
|
||||||
|
return service.DatasetGroupInput{Code: r.Code, Name: r.Name, Description: r.Description}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *DatasetGroupHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||||
|
req, err := httputil.DecodeJSON[datasetGroupRequest](w, r)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "invalid request body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := h.validate.Struct(req); err != nil {
|
||||||
|
httputil.WriteValidationErrors(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
group, err := h.svc.Create(r.Context(), req.toInput())
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusCreated, group)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *DatasetGroupHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var code *string
|
||||||
|
if v := strings.TrimSpace(r.URL.Query().Get("code")); v != "" {
|
||||||
|
code = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
page, ok := parsePositiveIntQuery(w, r, "page", 1)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pageSize, ok := parsePositiveIntQuery(w, r, "page_size", service.DefaultPageSize)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.svc.List(r.Context(), code, page, pageSize)
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, newPaginated(result.Items, result.Page, result.PageSize, result.Total))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *DatasetGroupHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
group, err := h.svc.Get(r.Context(), id)
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, group)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *DatasetGroupHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req, err := httputil.DecodeJSON[datasetGroupRequest](w, r)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "invalid request body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := h.validate.Struct(req); err != nil {
|
||||||
|
httputil.WriteValidationErrors(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
group, err := h.svc.Update(r.Context(), id, req.toInput())
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, group)
|
||||||
|
}
|
||||||
|
|
||||||
|
// delete removes a group. A group still referenced by datasets yields 409.
|
||||||
|
func (h *DatasetGroupHandler) delete(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := h.svc.Delete(r.Context(), id); err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// latest returns the freshest dataset of the group (greatest end_date, NULLs
|
||||||
|
// last, then newest), optionally narrowed by ?status= and ?file_type=.
|
||||||
|
func (h *DatasetGroupHandler) latest(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var status *string
|
||||||
|
if v := strings.TrimSpace(r.URL.Query().Get("status")); v != "" {
|
||||||
|
if !domain.ValidDatasetStatus(v) {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "invalid status")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
status = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
var fileType *domain.FileType
|
||||||
|
if v := strings.TrimSpace(r.URL.Query().Get("file_type")); v != "" {
|
||||||
|
ft := domain.FileType(v)
|
||||||
|
if !ft.Valid() {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "file_type must be one of: vector_with_kato, vector, raster")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fileType = &ft
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset, err := h.svc.Latest(r.Context(), id, status, fileType)
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, dataset)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *DatasetGroupHandler) uploadSLD(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
file, header, err := r.FormFile("file")
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "file is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
contentType := header.Header.Get("Content-Type")
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/xml"
|
||||||
|
}
|
||||||
|
|
||||||
|
group, err := h.svc.UploadSLD(r.Context(), id, service.UploadSLDInput{
|
||||||
|
Filename: header.Filename,
|
||||||
|
ContentType: contentType,
|
||||||
|
Size: header.Size,
|
||||||
|
Reader: file,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, group)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *DatasetGroupHandler) downloadSLD(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
group, obj, err := h.svc.DownloadSLD(r.Context(), id)
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer obj.Close()
|
||||||
|
|
||||||
|
contentType := "application/xml"
|
||||||
|
if group.SldContentType != nil && *group.SldContentType != "" {
|
||||||
|
contentType = *group.SldContentType
|
||||||
|
}
|
||||||
|
filename := "style.sld"
|
||||||
|
if group.SldFilename != nil && *group.SldFilename != "" {
|
||||||
|
filename = *group.SldFilename
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", contentType)
|
||||||
|
w.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`)
|
||||||
|
if _, err := io.Copy(w, obj); err != nil {
|
||||||
|
// Headers are already sent; nothing useful to return to the client.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ package http
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -36,10 +37,14 @@ func (h *DatasetHandler) Register(r chi.Router) {
|
|||||||
r.Get("/", h.list)
|
r.Get("/", h.list)
|
||||||
r.Post("/", h.upload)
|
r.Post("/", h.upload)
|
||||||
r.Get("/{id}", h.get)
|
r.Get("/{id}", h.get)
|
||||||
|
r.Put("/{id}", h.update)
|
||||||
|
r.Patch("/{id}", h.patch)
|
||||||
r.Get("/{id}.geojson", h.geojson)
|
r.Get("/{id}.geojson", h.geojson)
|
||||||
r.Get("/{id}.kato.geojson", h.katoGeoJSON)
|
r.Get("/{id}.cog", h.cog)
|
||||||
r.Get("/{id}/status", h.status)
|
r.Get("/{id}/status", h.status)
|
||||||
r.Get("/{id}/download", h.download)
|
r.Get("/{id}/download", h.download)
|
||||||
|
r.Post("/{id}/sld", h.uploadSLD)
|
||||||
|
r.Get("/{id}/sld", h.downloadSLD)
|
||||||
r.Post("/{id}/mapping", h.mapping)
|
r.Post("/{id}/mapping", h.mapping)
|
||||||
r.Get("/{id}/observations", h.observations)
|
r.Get("/{id}/observations", h.observations)
|
||||||
r.Delete("/{id}", h.delete)
|
r.Delete("/{id}", h.delete)
|
||||||
@@ -75,8 +80,9 @@ func (h *DatasetHandler) status(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type yearColumnInput struct {
|
type yearColumnInput struct {
|
||||||
Column string `json:"column" validate:"required"`
|
Column string `json:"column" validate:"required"`
|
||||||
Date string `json:"date" validate:"required,datetime=2006-01-02"`
|
StartDate string `json:"start_date" validate:"required,datetime=2006-01-02"`
|
||||||
|
EndDate *string `json:"end_date" validate:"omitempty,datetime=2006-01-02"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type mappingRequest struct {
|
type mappingRequest struct {
|
||||||
@@ -103,7 +109,7 @@ func (h *DatasetHandler) mapping(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
in := service.MappingInput{KatoColumn: req.KatoColumn}
|
in := service.MappingInput{KatoColumn: req.KatoColumn}
|
||||||
for _, yc := range req.YearColumns {
|
for _, yc := range req.YearColumns {
|
||||||
in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, Date: yc.Date})
|
in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, StartDate: yc.StartDate, EndDate: yc.EndDate})
|
||||||
}
|
}
|
||||||
|
|
||||||
dataset, err := h.svc.SaveMapping(r.Context(), id, in)
|
dataset, err := h.svc.SaveMapping(r.Context(), id, in)
|
||||||
@@ -144,6 +150,17 @@ func (h *DatasetHandler) upload(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The dataset group is optional; the period is validated by the service.
|
||||||
|
var groupID *uuid.UUID
|
||||||
|
if v := optionalFormValue(r, "dg_id"); v != nil {
|
||||||
|
gid, err := uuid.Parse(*v)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "dg_id must be a valid UUID")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
groupID = &gid
|
||||||
|
}
|
||||||
|
|
||||||
file, header, err := r.FormFile("file")
|
file, header, err := r.FormFile("file")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
httputil.WriteError(w, http.StatusBadRequest, "file is required")
|
httputil.WriteError(w, http.StatusBadRequest, "file is required")
|
||||||
@@ -157,17 +174,20 @@ func (h *DatasetHandler) upload(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
dataset, err := h.svc.Upload(r.Context(), service.UploadInput{
|
dataset, err := h.svc.Upload(r.Context(), service.UploadInput{
|
||||||
CategoryID: categoryID,
|
CategoryID: categoryID,
|
||||||
Name: r.FormValue("name"),
|
Name: r.FormValue("name"),
|
||||||
Description: optionalFormValue(r, "description"),
|
Description: optionalFormValue(r, "description"),
|
||||||
Unit: optionalFormValue(r, "unit"),
|
Unit: optionalFormValue(r, "unit"),
|
||||||
Meta: meta,
|
Meta: meta,
|
||||||
Automated: automated,
|
Automated: automated,
|
||||||
Filename: header.Filename,
|
Filename: header.Filename,
|
||||||
FileType: fileType,
|
FileType: fileType,
|
||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
Size: header.Size,
|
Size: header.Size,
|
||||||
Reader: file,
|
Reader: file,
|
||||||
|
DatasetGroupID: groupID,
|
||||||
|
StartDate: optionalFormValue(r, "start_date"),
|
||||||
|
EndDate: optionalFormValue(r, "end_date"),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
respondDomainError(w, err)
|
respondDomainError(w, err)
|
||||||
@@ -220,7 +240,32 @@ func (h *DatasetHandler) list(w http.ResponseWriter, r *http.Request) {
|
|||||||
categoryCode = &v
|
categoryCode = &v
|
||||||
}
|
}
|
||||||
|
|
||||||
filter := domain.DatasetFilter{CategoryID: categoryID}
|
groupID, ok := parseOptionalUUIDQuery(w, r, "dg_id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
filter := domain.DatasetFilter{CategoryID: categoryID, DatasetGroupID: groupID}
|
||||||
|
|
||||||
|
for _, b := range []struct {
|
||||||
|
name string
|
||||||
|
dst **string
|
||||||
|
}{
|
||||||
|
{"start_date_from", &filter.StartDateFrom},
|
||||||
|
{"start_date_to", &filter.StartDateTo},
|
||||||
|
{"end_date_from", &filter.EndDateFrom},
|
||||||
|
{"end_date_to", &filter.EndDateTo},
|
||||||
|
} {
|
||||||
|
v := strings.TrimSpace(r.URL.Query().Get(b.name))
|
||||||
|
if v == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := time.Parse("2006-01-02", v); err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, b.name+" must be YYYY-MM-DD")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
*b.dst = &v
|
||||||
|
}
|
||||||
|
|
||||||
if v := strings.TrimSpace(r.URL.Query().Get("file_type")); v != "" {
|
if v := strings.TrimSpace(r.URL.Query().Get("file_type")); v != "" {
|
||||||
ft := domain.FileType(v)
|
ft := domain.FileType(v)
|
||||||
@@ -305,10 +350,227 @@ func (h *DatasetHandler) get(w http.ResponseWriter, r *http.Request) {
|
|||||||
httputil.WriteJSON(w, http.StatusOK, dataset)
|
httputil.WriteJSON(w, http.StatusOK, dataset)
|
||||||
}
|
}
|
||||||
|
|
||||||
// geojson returns the dataset as a GeoJSON FeatureCollection (RFC 7946). For a
|
// datasetPutRequest is the full JSON body for PUT /datasets/{id}: every
|
||||||
// vector_with_kato dataset it serves the dataset's own geometry as a single
|
// editable field is required. Description and Unit are pointers so an empty
|
||||||
// feature when present, otherwise one feature per KATO joined to the districts
|
// string is distinguishable from an absent field; a blank value clears the
|
||||||
// table. Only vector_with_kato datasets are supported.
|
// (nullable) column. File-related and lifecycle fields are not editable here.
|
||||||
|
type datasetPutRequest struct {
|
||||||
|
CategoryID string `json:"category_id" validate:"required,uuid"`
|
||||||
|
Name string `json:"name" validate:"required,max=255"`
|
||||||
|
Description *string `json:"description" validate:"required"`
|
||||||
|
Unit *string `json:"unit" validate:"required"`
|
||||||
|
// DgID, StartDate and EndDate are optional; omitting them (or sending null
|
||||||
|
// or a blank string) clears the corresponding nullable column.
|
||||||
|
DgID *string `json:"dg_id" validate:"omitempty,uuid"`
|
||||||
|
StartDate *string `json:"start_date" validate:"omitempty,datetime=2006-01-02"`
|
||||||
|
EndDate *string `json:"end_date" validate:"omitempty,datetime=2006-01-02"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// update replaces a dataset's editable metadata (category, name, description,
|
||||||
|
// unit, dataset group, period).
|
||||||
|
func (h *DatasetHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req, err := httputil.DecodeJSON[datasetPutRequest](w, r)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "invalid request body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := h.validate.Struct(req); err != nil {
|
||||||
|
httputil.WriteValidationErrors(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
categoryID, err := uuid.Parse(req.CategoryID)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "category_id must be a valid UUID")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var groupID *uuid.UUID
|
||||||
|
if raw := optionalPtr(req.DgID); raw != nil {
|
||||||
|
gid, err := uuid.Parse(*raw)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "dg_id must be a valid UUID")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
groupID = &gid
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset, err := h.svc.Update(r.Context(), id, service.DatasetUpdateInput{
|
||||||
|
CategoryID: categoryID,
|
||||||
|
Name: strings.TrimSpace(req.Name),
|
||||||
|
Description: normalizeOptional(*req.Description),
|
||||||
|
Unit: normalizeOptional(*req.Unit),
|
||||||
|
DatasetGroupID: groupID,
|
||||||
|
StartDate: optionalPtr(req.StartDate),
|
||||||
|
EndDate: optionalPtr(req.EndDate),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, dataset)
|
||||||
|
}
|
||||||
|
|
||||||
|
// datasetPatchFields lists the JSON keys accepted by PATCH /datasets/{id}.
|
||||||
|
var datasetPatchFields = map[string]struct{}{
|
||||||
|
"category_id": {}, "name": {}, "description": {}, "unit": {},
|
||||||
|
"dg_id": {}, "start_date": {}, "end_date": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
// patch applies a sparse update to a dataset's editable metadata (category,
|
||||||
|
// name, description, unit, dg_id, start_date, end_date): only fields present in
|
||||||
|
// the body are changed. Sending an explicit null for a nullable field clears
|
||||||
|
// that column.
|
||||||
|
func (h *DatasetHandler) patch(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
raw, err := httputil.DecodeJSON[map[string]json.RawMessage](w, r)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "invalid request body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(raw) == 0 {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "request body must include at least one field")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var in service.DatasetPatchInput
|
||||||
|
for key, val := range raw {
|
||||||
|
if _, ok := datasetPatchFields[key]; !ok {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "unknown field: "+key)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch key {
|
||||||
|
case "category_id":
|
||||||
|
var s string
|
||||||
|
if err := json.Unmarshal(val, &s); err != nil || s == "" {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "category_id must be a valid UUID")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cid, err := uuid.Parse(s)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "category_id must be a valid UUID")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
in.CategoryID = &cid
|
||||||
|
case "name":
|
||||||
|
var s string
|
||||||
|
if err := json.Unmarshal(val, &s); err != nil || strings.TrimSpace(s) == "" {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "name must not be blank")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
name := strings.TrimSpace(s)
|
||||||
|
in.Name = &name
|
||||||
|
case "description":
|
||||||
|
desc, err := decodeOptionalPatchString(val)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "description must be a string or null")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
in.Description, in.DescriptionSet = desc, true
|
||||||
|
case "unit":
|
||||||
|
unit, err := decodeOptionalPatchString(val)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "unit must be a string or null")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
in.Unit, in.UnitSet = unit, true
|
||||||
|
case "dg_id":
|
||||||
|
s, err := decodeOptionalPatchString(val)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "dg_id must be a valid UUID or null")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if s != nil {
|
||||||
|
gid, err := uuid.Parse(*s)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "dg_id must be a valid UUID or null")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
in.DatasetGroupID = &gid
|
||||||
|
}
|
||||||
|
in.DatasetGroupIDSet = true
|
||||||
|
case "start_date":
|
||||||
|
date, err := decodeOptionalPatchDate(val)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "start_date must be YYYY-MM-DD or null")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
in.StartDate, in.StartDateSet = date, true
|
||||||
|
case "end_date":
|
||||||
|
date, err := decodeOptionalPatchDate(val)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "end_date must be YYYY-MM-DD or null")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
in.EndDate, in.EndDateSet = date, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset, err := h.svc.Patch(r.Context(), id, in)
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, dataset)
|
||||||
|
}
|
||||||
|
|
||||||
|
// optionalPtr normalizes an optional request field: a nil or blank value
|
||||||
|
// yields nil, so an omitted or empty field clears a nullable column.
|
||||||
|
func optionalPtr(v *string) *string {
|
||||||
|
if v == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return normalizeOptional(*v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeOptional trims s and returns nil when blank, so an empty string
|
||||||
|
// clears a nullable column rather than being stored verbatim.
|
||||||
|
func normalizeOptional(s string) *string {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &s
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeOptionalPatchString unmarshals a PATCH field that may be a JSON string
|
||||||
|
// or null, returning nil (clear the column) for null or a blank string.
|
||||||
|
func decodeOptionalPatchString(raw json.RawMessage) (*string, error) {
|
||||||
|
if string(raw) == "null" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var s string
|
||||||
|
if err := json.Unmarshal(raw, &s); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return normalizeOptional(s), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeOptionalPatchDate unmarshals a PATCH date field that may be a
|
||||||
|
// YYYY-MM-DD JSON string or null, returning nil (clear the column) for null or
|
||||||
|
// a blank string.
|
||||||
|
func decodeOptionalPatchDate(raw json.RawMessage) (*string, error) {
|
||||||
|
v, err := decodeOptionalPatchString(raw)
|
||||||
|
if err != nil || v == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, err := time.Parse("2006-01-02", *v); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// geojson returns the dataset's pre-assembled GeoJSON FeatureCollection (RFC
|
||||||
|
// 7946), generated and stored at processing time. A vector_with_kato dataset's
|
||||||
|
// collection joins the districts table on KATO code (one feature per KATO with
|
||||||
|
// the observation values mapped onto its district polygon); a plain vector
|
||||||
|
// dataset's collection wraps its own geometry as a single feature.
|
||||||
func (h *DatasetHandler) geojson(w http.ResponseWriter, r *http.Request) {
|
func (h *DatasetHandler) geojson(w http.ResponseWriter, r *http.Request) {
|
||||||
id, ok := parseUUIDParam(w, r, "id")
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -321,26 +583,45 @@ func (h *DatasetHandler) geojson(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
w.Header().Set("Content-Type", "application/geo+json")
|
w.Header().Set("Content-Type", "application/geo+json")
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
_ = json.NewEncoder(w).Encode(fc)
|
_, _ = w.Write(fc)
|
||||||
}
|
}
|
||||||
|
|
||||||
// katoGeoJSON returns the dataset as a GeoJSON FeatureCollection (RFC 7946),
|
// cog streams the raster dataset's Cloud-Optimized GeoTIFF. It responds 422 for
|
||||||
// ignoring any geometry the dataset carries and instead joining the districts
|
// a non-raster dataset and 409 when the raster's COG has not been produced yet.
|
||||||
// table on KATO code: one feature per KATO with the observation values mapped
|
func (h *DatasetHandler) cog(w http.ResponseWriter, r *http.Request) {
|
||||||
// onto its district polygon. Only vector_with_kato datasets are supported.
|
|
||||||
func (h *DatasetHandler) katoGeoJSON(w http.ResponseWriter, r *http.Request) {
|
|
||||||
id, ok := parseUUIDParam(w, r, "id")
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fc, err := h.svc.KatoGeoJSON(r.Context(), id)
|
dataset, obj, err := h.svc.COG(r.Context(), id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
respondDomainError(w, err)
|
switch {
|
||||||
|
case errors.Is(err, service.ErrNotRaster):
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "dataset is not a raster")
|
||||||
|
case errors.Is(err, service.ErrCOGNotReady):
|
||||||
|
httputil.WriteError(w, http.StatusConflict, "cog is not ready")
|
||||||
|
default:
|
||||||
|
respondDomainError(w, err)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Content-Type", "application/geo+json")
|
defer obj.Close()
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "image/tiff")
|
||||||
|
w.Header().Set("Content-Disposition", `inline; filename="`+cogFilename(dataset.Filename)+`"`)
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
_ = json.NewEncoder(w).Encode(fc)
|
if _, err := io.Copy(w, obj); err != nil {
|
||||||
|
// Headers are already sent; nothing useful to return to the client.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cogFilename derives a .cog.tif download name from the source filename.
|
||||||
|
func cogFilename(filename string) string {
|
||||||
|
if i := strings.LastIndex(filename, "."); i > 0 {
|
||||||
|
filename = filename[:i]
|
||||||
|
}
|
||||||
|
return filename + ".cog.tif"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *DatasetHandler) download(w http.ResponseWriter, r *http.Request) {
|
func (h *DatasetHandler) download(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -366,6 +647,74 @@ func (h *DatasetHandler) download(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// uploadSLD stores (or replaces) a dataset's SLD (Styled Layer Descriptor)
|
||||||
|
// style file. Multipart upload, mirroring POST /datasets. Applies to every
|
||||||
|
// file_type.
|
||||||
|
func (h *DatasetHandler) uploadSLD(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
file, header, err := r.FormFile("file")
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "file is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
contentType := header.Header.Get("Content-Type")
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/xml"
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset, err := h.svc.UploadSLD(r.Context(), id, service.UploadSLDInput{
|
||||||
|
Filename: header.Filename,
|
||||||
|
ContentType: contentType,
|
||||||
|
Size: header.Size,
|
||||||
|
Reader: file,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, dataset)
|
||||||
|
}
|
||||||
|
|
||||||
|
// downloadSLD streams a dataset's stored SLD style file. Returns 404 if none
|
||||||
|
// has been uploaded yet.
|
||||||
|
func (h *DatasetHandler) downloadSLD(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
dataset, obj, err := h.svc.DownloadSLD(r.Context(), id)
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer obj.Close()
|
||||||
|
|
||||||
|
contentType := "application/xml"
|
||||||
|
if dataset.SldContentType != nil && *dataset.SldContentType != "" {
|
||||||
|
contentType = *dataset.SldContentType
|
||||||
|
}
|
||||||
|
filename := "style.sld"
|
||||||
|
if dataset.SldFilename != nil && *dataset.SldFilename != "" {
|
||||||
|
filename = *dataset.SldFilename
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", contentType)
|
||||||
|
w.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`)
|
||||||
|
if _, err := io.Copy(w, obj); err != nil {
|
||||||
|
// Headers are already sent; nothing useful to return to the client.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (h *DatasetHandler) delete(w http.ResponseWriter, r *http.Request) {
|
func (h *DatasetHandler) delete(w http.ResponseWriter, r *http.Request) {
|
||||||
id, ok := parseUUIDParam(w, r, "id")
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|||||||
186
internal/transport/http/landscape_handler.go
Normal file
186
internal/transport/http/landscape_handler.go
Normal file
@@ -0,0 +1,186 @@
|
|||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"gis/internal/landscape"
|
||||||
|
"gis/internal/service"
|
||||||
|
"gis/pkg/httputil"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
"github.com/go-playground/validator/v10"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LandscapeHandler serves the /landscape routes.
|
||||||
|
type LandscapeHandler struct {
|
||||||
|
svc *service.LandscapeService
|
||||||
|
validate *validator.Validate
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLandscapeHandler returns a LandscapeHandler.
|
||||||
|
func NewLandscapeHandler(svc *service.LandscapeService, validate *validator.Validate) *LandscapeHandler {
|
||||||
|
return &LandscapeHandler{svc: svc, validate: validate}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register mounts the Landscape Program routes on r.
|
||||||
|
func (h *LandscapeHandler) Register(r chi.Router) {
|
||||||
|
r.Get("/slots", h.slots)
|
||||||
|
r.Post("/slots/{code}", h.uploadSlot)
|
||||||
|
r.Get("/runs", h.listRuns)
|
||||||
|
r.Post("/runs", h.createRun)
|
||||||
|
r.Get("/runs/{id}", h.getRun)
|
||||||
|
r.Get("/results", h.results)
|
||||||
|
}
|
||||||
|
|
||||||
|
// runRequest is the body of POST /landscape/runs. Overrides are decoded as
|
||||||
|
// strings so a malformed id is a 422 instead of a 400 on the whole body.
|
||||||
|
type runRequest struct {
|
||||||
|
FromStage *int `json:"from_stage" validate:"omitempty,gte=0,lte=3"`
|
||||||
|
Overrides map[string]string `json:"overrides" validate:"omitempty,dive,uuid"`
|
||||||
|
Params *landscape.Params `json:"params"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// slots lists every leaf input slot with the dataset currently filling it.
|
||||||
|
func (h *LandscapeHandler) slots(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slots, err := h.svc.Slots(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, slots)
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadSlot stores a raster in the slot's dataset group under the landscape
|
||||||
|
// category. An unknown slot code yields 404.
|
||||||
|
func (h *LandscapeHandler) uploadSlot(w http.ResponseWriter, r *http.Request) {
|
||||||
|
code := chi.URLParam(r, "code")
|
||||||
|
|
||||||
|
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
file, header, err := r.FormFile("file")
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "file is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
contentType := header.Header.Get("Content-Type")
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "image/tiff"
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset, err := h.svc.UploadToSlot(r.Context(), code, service.SlotUploadInput{
|
||||||
|
Filename: header.Filename,
|
||||||
|
ContentType: contentType,
|
||||||
|
Size: header.Size,
|
||||||
|
Reader: file,
|
||||||
|
Name: r.FormValue("name"),
|
||||||
|
StartDate: optionalFormValue(r, "start_date"),
|
||||||
|
EndDate: optionalFormValue(r, "end_date"),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusCreated, dataset)
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeRunRequest reads the run body. The whole body is optional — every field
|
||||||
|
// has a default — so an empty body is the zero request (a default incremental
|
||||||
|
// run), not an error. A malformed body writes a 400 and reports false.
|
||||||
|
func decodeRunRequest(w http.ResponseWriter, r *http.Request) (runRequest, bool) {
|
||||||
|
req, err := httputil.DecodeJSON[runRequest](w, r)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, io.EOF):
|
||||||
|
return runRequest{}, true
|
||||||
|
case err != nil:
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "invalid request body")
|
||||||
|
return runRequest{}, false
|
||||||
|
}
|
||||||
|
return req, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// createRun queues a Landscape Program run and returns it with 202 Accepted;
|
||||||
|
// the worker executes it asynchronously.
|
||||||
|
func (h *LandscapeHandler) createRun(w http.ResponseWriter, r *http.Request) {
|
||||||
|
req, ok := decodeRunRequest(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := h.validate.Struct(req); err != nil {
|
||||||
|
httputil.WriteValidationErrors(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
overrides := make(map[string]uuid.UUID, len(req.Overrides))
|
||||||
|
for group, raw := range req.Overrides {
|
||||||
|
id, err := uuid.Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusUnprocessableEntity, "overrides["+group+"] must be a valid UUID")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
overrides[group] = id
|
||||||
|
}
|
||||||
|
|
||||||
|
in := service.RunInput{FromStage: req.FromStage, Overrides: overrides}
|
||||||
|
if req.Params != nil {
|
||||||
|
in.Params = *req.Params
|
||||||
|
}
|
||||||
|
|
||||||
|
run, err := h.svc.CreateRun(r.Context(), in)
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusAccepted, run)
|
||||||
|
}
|
||||||
|
|
||||||
|
// listRuns returns a page of runs, newest first.
|
||||||
|
func (h *LandscapeHandler) listRuns(w http.ResponseWriter, r *http.Request) {
|
||||||
|
page, ok := parsePositiveIntQuery(w, r, "page", 1)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pageSize, ok := parsePositiveIntQuery(w, r, "page_size", service.DefaultPageSize)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.svc.ListRuns(r.Context(), page, pageSize)
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, newPaginated(result.Items, result.Page, result.PageSize, result.Total))
|
||||||
|
}
|
||||||
|
|
||||||
|
// getRun returns one run with its per-node execution state.
|
||||||
|
func (h *LandscapeHandler) getRun(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
run, err := h.svc.GetRun(r.Context(), id)
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, run)
|
||||||
|
}
|
||||||
|
|
||||||
|
// results lists every output group with its freshest ready dataset and the URLs
|
||||||
|
// to render it.
|
||||||
|
func (h *LandscapeHandler) results(w http.ResponseWriter, r *http.Request) {
|
||||||
|
results, err := h.svc.Results(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, results)
|
||||||
|
}
|
||||||
191
internal/transport/http/landscape_handler_test.go
Normal file
191
internal/transport/http/landscape_handler_test.go
Normal file
@@ -0,0 +1,191 @@
|
|||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
"github.com/go-playground/validator/v10"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newLandscapeTestHandler builds a handler with a nil service: every case below
|
||||||
|
// is rejected while parsing or validating the request, before the service is
|
||||||
|
// reached.
|
||||||
|
func newLandscapeTestHandler() *LandscapeHandler {
|
||||||
|
return NewLandscapeHandler(nil, validator.New(validator.WithRequiredStructEnabled()))
|
||||||
|
}
|
||||||
|
|
||||||
|
func postRun(t *testing.T, body string) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/landscape/runs", strings.NewReader(body))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
newLandscapeTestHandler().createRun(rec, req)
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCreateRunInvalidJSON(t *testing.T) {
|
||||||
|
rec := postRun(t, `{"from_stage":`)
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want %d (body %s)", rec.Code, http.StatusBadRequest, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCreateRunUnknownField(t *testing.T) {
|
||||||
|
// DecodeJSON rejects unknown fields, so a typo is a 400 rather than a
|
||||||
|
// silently ignored option.
|
||||||
|
rec := postRun(t, `{"from_stages":1}`)
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want %d (body %s)", rec.Code, http.StatusBadRequest, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCreateRunFromStageOutOfRange(t *testing.T) {
|
||||||
|
rec := postRun(t, `{"from_stage":7}`)
|
||||||
|
if rec.Code != http.StatusUnprocessableEntity {
|
||||||
|
t.Fatalf("status = %d, want %d (body %s)", rec.Code, http.StatusUnprocessableEntity, rec.Body.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(rec.Body.String(), "FromStage") {
|
||||||
|
t.Fatalf("body %s does not name the offending field", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCreateRunOverrideNotUUID(t *testing.T) {
|
||||||
|
rec := postRun(t, `{"overrides":{"lp-z-lcr":"not-a-uuid"}}`)
|
||||||
|
if rec.Code != http.StatusUnprocessableEntity {
|
||||||
|
t.Fatalf("status = %d, want %d (body %s)", rec.Code, http.StatusUnprocessableEntity, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadSlotRequiresMultipart(t *testing.T) {
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/landscape/slots/lp-z-lcr", strings.NewReader(`{}`))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
newLandscapeTestHandler().uploadSlot(rec, req)
|
||||||
|
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want %d (body %s)", rec.Code, http.StatusBadRequest, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListRunsInvalidPage(t *testing.T) {
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/landscape/runs?page=0", nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
newLandscapeTestHandler().listRuns(rec, req)
|
||||||
|
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want %d (body %s)", rec.Code, http.StatusBadRequest, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetRunInvalidID(t *testing.T) {
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/landscape/runs/nope", nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
newLandscapeTestHandler().getRun(rec, req)
|
||||||
|
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want %d (body %s)", rec.Code, http.StatusBadRequest, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDecodeRunRequestEmptyBody pins the "all defaults" call: POST
|
||||||
|
// /landscape/runs with no body at all is a valid request, not a 400. Tested at
|
||||||
|
// the helper level because the 202 path needs a live service.
|
||||||
|
func TestDecodeRunRequestEmptyBody(t *testing.T) {
|
||||||
|
for _, body := range []string{"", "{}"} {
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/landscape/runs", strings.NewReader(body))
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
|
||||||
|
got, ok := decodeRunRequest(rec, req)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("body %q: decodeRunRequest reported failure with status %d", body, rec.Code)
|
||||||
|
}
|
||||||
|
if got.FromStage != nil || len(got.Overrides) != 0 || got.Params != nil {
|
||||||
|
t.Fatalf("body %q: got %+v, want the zero request", body, got)
|
||||||
|
}
|
||||||
|
if rec.Code != http.StatusOK || rec.Body.Len() != 0 {
|
||||||
|
t.Fatalf("body %q: helper wrote a response (status %d, body %s)", body, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
// The zero request must also survive validation, or an empty body
|
||||||
|
// would still be rejected one step later.
|
||||||
|
if err := validator.New(validator.WithRequiredStructEnabled()).Struct(got); err != nil {
|
||||||
|
t.Fatalf("body %q: zero request fails validation: %v", body, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeRunRequestMalformedBody(t *testing.T) {
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/landscape/runs", strings.NewReader(`{`))
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
|
||||||
|
if _, ok := decodeRunRequest(rec, req); ok {
|
||||||
|
t.Fatal("decodeRunRequest accepted a truncated body")
|
||||||
|
}
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRegisterMountsRoutes walks the route table of a router with the handler
|
||||||
|
// registered, so every one of the six routes is checked — including GET /slots
|
||||||
|
// and GET /results, which cannot be exercised without a live service.
|
||||||
|
func TestRegisterMountsRoutes(t *testing.T) {
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Route("/landscape", newLandscapeTestHandler().Register)
|
||||||
|
|
||||||
|
got := map[string]bool{}
|
||||||
|
err := chi.Walk(r, func(method, route string, _ http.Handler, _ ...func(http.Handler) http.Handler) error {
|
||||||
|
got[method+" "+strings.TrimSuffix(route, "/")] = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("walk routes: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{
|
||||||
|
"GET /landscape/slots",
|
||||||
|
"POST /landscape/slots/{code}",
|
||||||
|
"GET /landscape/runs",
|
||||||
|
"POST /landscape/runs",
|
||||||
|
"GET /landscape/runs/{id}",
|
||||||
|
"GET /landscape/results",
|
||||||
|
}
|
||||||
|
for _, w := range want {
|
||||||
|
if !got[w] {
|
||||||
|
t.Errorf("route %q is not mounted; mounted routes: %v", w, keys(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Errorf("mounted %d routes, want %d: %v", len(got), len(want), keys(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRoutedRequestReachesHandler sends a real request through the router so the
|
||||||
|
// URL parameter wiring (chi.URLParam) is exercised, not just the pattern.
|
||||||
|
func TestRoutedRequestReachesHandler(t *testing.T) {
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Route("/landscape", newLandscapeTestHandler().Register)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/landscape/runs/nope", nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
r.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want %d (body %s)", rec.Code, http.StatusBadRequest, rec.Body.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(rec.Body.String(), "invalid id") {
|
||||||
|
t.Fatalf("body %s does not report the bad id", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func keys(m map[string]bool) []string {
|
||||||
|
out := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -10,10 +10,14 @@ import (
|
|||||||
|
|
||||||
// RouterDeps holds the handlers and dependencies the router mounts.
|
// RouterDeps holds the handlers and dependencies the router mounts.
|
||||||
type RouterDeps struct {
|
type RouterDeps struct {
|
||||||
Logger *slog.Logger
|
Logger *slog.Logger
|
||||||
Health *HealthHandler
|
Health *HealthHandler
|
||||||
Categories *CategoryHandler
|
Categories *CategoryHandler
|
||||||
Datasets *DatasetHandler
|
Datasets *DatasetHandler
|
||||||
|
// DatasetGroups serves the /dataset-groups routes.
|
||||||
|
DatasetGroups *DatasetGroupHandler
|
||||||
|
// Landscape serves the /landscape routes.
|
||||||
|
Landscape *LandscapeHandler
|
||||||
OpenAPISpec []byte
|
OpenAPISpec []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -37,6 +41,8 @@ func NewRouter(deps RouterDeps) http.Handler {
|
|||||||
// Domain routes. Add future feature routers here.
|
// Domain routes. Add future feature routers here.
|
||||||
r.Route("/categories", deps.Categories.Register)
|
r.Route("/categories", deps.Categories.Register)
|
||||||
r.Route("/datasets", deps.Datasets.Register)
|
r.Route("/datasets", deps.Datasets.Register)
|
||||||
|
r.Route("/dataset-groups", deps.DatasetGroups.Register)
|
||||||
|
r.Route("/landscape", deps.Landscape.Register)
|
||||||
|
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|||||||
8
migrations/00007_add_geojson_to_datasets.sql
Normal file
8
migrations/00007_add_geojson_to_datasets.sql
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
-- +goose Up
|
||||||
|
-- Pre-assembled GeoJSON FeatureCollection served by the /datasets/{id}.geojson
|
||||||
|
-- endpoint. Generated at processing time for vector and vector_with_kato
|
||||||
|
-- datasets (NULL until generated / for rasters).
|
||||||
|
ALTER TABLE datasets ADD COLUMN geojson JSONB;
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
ALTER TABLE datasets DROP COLUMN geojson;
|
||||||
12
migrations/00008_add_sld_to_datasets.sql
Normal file
12
migrations/00008_add_sld_to_datasets.sql
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
-- +goose Up
|
||||||
|
-- Optional SLD (Styled Layer Descriptor) style file for a dataset. Applies to
|
||||||
|
-- every file_type; a dataset may have at most one (uploading again replaces
|
||||||
|
-- it).
|
||||||
|
ALTER TABLE datasets ADD COLUMN sld_storage_key TEXT;
|
||||||
|
ALTER TABLE datasets ADD COLUMN sld_filename VARCHAR(255);
|
||||||
|
ALTER TABLE datasets ADD COLUMN sld_content_type VARCHAR(255);
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
ALTER TABLE datasets DROP COLUMN sld_storage_key;
|
||||||
|
ALTER TABLE datasets DROP COLUMN sld_filename;
|
||||||
|
ALTER TABLE datasets DROP COLUMN sld_content_type;
|
||||||
15
migrations/00009_add_period_to_dataset_observations.sql
Normal file
15
migrations/00009_add_period_to_dataset_observations.sql
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
-- +goose Up
|
||||||
|
-- Optional period end date, so a year_column mapping can represent a date
|
||||||
|
-- range (date..date_to) instead of a single point in time. NULL preserves the
|
||||||
|
-- pre-existing point-in-time behavior; the unique index treats a NULL date_to
|
||||||
|
-- as equal to date itself so point observations keep deduplicating exactly as
|
||||||
|
-- before.
|
||||||
|
ALTER TABLE dataset_observations ADD COLUMN date_to DATE;
|
||||||
|
|
||||||
|
DROP INDEX uq_dataset_observations;
|
||||||
|
CREATE UNIQUE INDEX uq_dataset_observations ON dataset_observations (dataset_id, kato_code, date, COALESCE(date_to, date));
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
DROP INDEX uq_dataset_observations;
|
||||||
|
ALTER TABLE dataset_observations DROP COLUMN date_to;
|
||||||
|
CREATE UNIQUE INDEX uq_dataset_observations ON dataset_observations (dataset_id, kato_code, date);
|
||||||
67
migrations/00010_dataset_groups_and_dates.sql
Normal file
67
migrations/00010_dataset_groups_and_dates.sql
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
-- +goose Up
|
||||||
|
-- Dataset groups: "одноимённые" datasets versioned by period. A group may carry a
|
||||||
|
-- shared SLD used as a fallback for its datasets.
|
||||||
|
CREATE TABLE dataset_groups (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
code VARCHAR(255) NOT NULL UNIQUE,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
description TEXT NOT NULL DEFAULT '',
|
||||||
|
sld_storage_key TEXT,
|
||||||
|
sld_filename VARCHAR(255),
|
||||||
|
sld_content_type VARCHAR(255),
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
ALTER TABLE datasets ADD COLUMN dg_id UUID REFERENCES dataset_groups (id) ON DELETE RESTRICT;
|
||||||
|
ALTER TABLE datasets ADD COLUMN start_date DATE;
|
||||||
|
ALTER TABLE datasets ADD COLUMN end_date DATE;
|
||||||
|
ALTER TABLE datasets ADD CONSTRAINT chk_datasets_period
|
||||||
|
CHECK (start_date IS NULL OR end_date IS NULL OR end_date >= start_date);
|
||||||
|
CREATE INDEX idx_datasets_dg_id_end_date ON datasets (dg_id, end_date DESC NULLS LAST, created_at DESC);
|
||||||
|
CREATE INDEX idx_datasets_start_date ON datasets (start_date);
|
||||||
|
CREATE INDEX idx_datasets_end_date ON datasets (end_date);
|
||||||
|
|
||||||
|
-- Period columns are renamed everywhere: observations, and the year_columns
|
||||||
|
-- mapping JSON ({column, date, date_to} -> {column, start_date, end_date}).
|
||||||
|
ALTER TABLE dataset_observations RENAME COLUMN date TO start_date;
|
||||||
|
ALTER TABLE dataset_observations RENAME COLUMN date_to TO end_date;
|
||||||
|
DROP INDEX uq_dataset_observations;
|
||||||
|
DROP INDEX idx_dataset_observations_kato_date;
|
||||||
|
CREATE UNIQUE INDEX uq_dataset_observations ON dataset_observations (dataset_id, kato_code, start_date, COALESCE(end_date, start_date));
|
||||||
|
CREATE INDEX idx_dataset_observations_kato_date ON dataset_observations (kato_code, start_date);
|
||||||
|
|
||||||
|
UPDATE datasets
|
||||||
|
SET year_columns = (
|
||||||
|
SELECT jsonb_agg(
|
||||||
|
(elem - 'date' - 'date_to')
|
||||||
|
|| jsonb_build_object('start_date', elem->'date')
|
||||||
|
|| CASE WHEN elem ? 'date_to' AND elem->'date_to' IS NOT NULL
|
||||||
|
THEN jsonb_build_object('end_date', elem->'date_to') ELSE '{}'::jsonb END)
|
||||||
|
FROM jsonb_array_elements(year_columns) AS elem)
|
||||||
|
WHERE year_columns IS NOT NULL AND jsonb_typeof(year_columns) = 'array' AND jsonb_array_length(year_columns) > 0;
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
UPDATE datasets
|
||||||
|
SET year_columns = (
|
||||||
|
SELECT jsonb_agg(
|
||||||
|
(elem - 'start_date' - 'end_date')
|
||||||
|
|| jsonb_build_object('date', elem->'start_date')
|
||||||
|
|| CASE WHEN elem ? 'end_date' AND elem->'end_date' IS NOT NULL
|
||||||
|
THEN jsonb_build_object('date_to', elem->'end_date') ELSE '{}'::jsonb END)
|
||||||
|
FROM jsonb_array_elements(year_columns) AS elem)
|
||||||
|
WHERE year_columns IS NOT NULL AND jsonb_typeof(year_columns) = 'array' AND jsonb_array_length(year_columns) > 0;
|
||||||
|
DROP INDEX uq_dataset_observations;
|
||||||
|
DROP INDEX idx_dataset_observations_kato_date;
|
||||||
|
ALTER TABLE dataset_observations RENAME COLUMN start_date TO date;
|
||||||
|
ALTER TABLE dataset_observations RENAME COLUMN end_date TO date_to;
|
||||||
|
CREATE UNIQUE INDEX uq_dataset_observations ON dataset_observations (dataset_id, kato_code, date, COALESCE(date_to, date));
|
||||||
|
CREATE INDEX idx_dataset_observations_kato_date ON dataset_observations (kato_code, date);
|
||||||
|
DROP INDEX idx_datasets_dg_id_end_date;
|
||||||
|
DROP INDEX idx_datasets_start_date;
|
||||||
|
DROP INDEX idx_datasets_end_date;
|
||||||
|
ALTER TABLE datasets DROP CONSTRAINT chk_datasets_period;
|
||||||
|
ALTER TABLE datasets DROP COLUMN dg_id;
|
||||||
|
ALTER TABLE datasets DROP COLUMN start_date;
|
||||||
|
ALTER TABLE datasets DROP COLUMN end_date;
|
||||||
|
DROP TABLE dataset_groups;
|
||||||
51
migrations/00011_landscape.sql
Normal file
51
migrations/00011_landscape.sql
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
-- +goose Up
|
||||||
|
-- Landscape Program: seed the category and dataset groups (input slots and
|
||||||
|
-- node outputs; codes must match internal/landscape/graph.go), and store runs.
|
||||||
|
INSERT INTO categories (code, name, description)
|
||||||
|
VALUES ('landscape', 'Ландшафтная программа', 'Входные и результирующие растры Ландшафтной программы')
|
||||||
|
ON CONFLICT (code) DO NOTHING;
|
||||||
|
|
||||||
|
INSERT INTO dataset_groups (code, name) VALUES
|
||||||
|
('lp-z-biodiversity', 'Z Biodiversity'), ('lp-z-lcr', 'Z LCR'), ('lp-z-soil-protection', 'Z Soil Protection'), ('lp-z-wr', 'Z Water Resources'),
|
||||||
|
('lp-u-surf-water', 'U Surface Water'), ('lp-u-soil', 'U Soil'), ('lp-u-crop', 'U Crop'), ('lp-u-veg', 'U Vegetation'),
|
||||||
|
('lp-p-irrigation', 'P Irrigation'), ('lp-p-snow', 'P Snow'), ('lp-p-crop', 'P Crop'), ('lp-p-pasture', 'P Pasture'), ('lp-p-tourism', 'P Tourism'),
|
||||||
|
('lp-z-integrated', 'Z Integrated'), ('lp-u-integrated', 'U Integrated'), ('lp-z-max', 'Z Max'), ('lp-u-max', 'U Max'),
|
||||||
|
('lp-e', 'E Ecological Constraint'),
|
||||||
|
('lp-c-irrigation', 'C Irrigation'), ('lp-c-snow', 'C Snow'), ('lp-c-crop', 'C Crop'), ('lp-c-pasture', 'C Pasture'), ('lp-c-tourism', 'C Tourism'),
|
||||||
|
('lp-p-max', 'P Max'),
|
||||||
|
('lp-v1-conservative', 'V1 Conservative'), ('lp-v2-balanced', 'V2 Balanced'), ('lp-v3-development', 'V3 Development'), ('lp-c-max', 'C Max')
|
||||||
|
ON CONFLICT (code) DO NOTHING;
|
||||||
|
|
||||||
|
CREATE TABLE lp_runs (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
status VARCHAR(32) NOT NULL,
|
||||||
|
from_stage SMALLINT,
|
||||||
|
params JSONB NOT NULL,
|
||||||
|
overrides JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||||
|
error TEXT,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
started_at TIMESTAMPTZ,
|
||||||
|
finished_at TIMESTAMPTZ
|
||||||
|
);
|
||||||
|
CREATE INDEX idx_lp_runs_created_at ON lp_runs (created_at DESC);
|
||||||
|
|
||||||
|
CREATE TABLE lp_run_nodes (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
run_id UUID NOT NULL REFERENCES lp_runs (id) ON DELETE CASCADE,
|
||||||
|
node_code VARCHAR(8) NOT NULL,
|
||||||
|
stage SMALLINT NOT NULL,
|
||||||
|
status VARCHAR(32) NOT NULL,
|
||||||
|
fingerprint TEXT NOT NULL DEFAULT '',
|
||||||
|
input_datasets JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||||
|
output_datasets JSONB,
|
||||||
|
error TEXT,
|
||||||
|
started_at TIMESTAMPTZ,
|
||||||
|
finished_at TIMESTAMPTZ,
|
||||||
|
UNIQUE (run_id, node_code)
|
||||||
|
);
|
||||||
|
CREATE INDEX idx_lp_run_nodes_lookup ON lp_run_nodes (node_code, fingerprint, status, finished_at DESC);
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
DROP TABLE lp_run_nodes;
|
||||||
|
DROP TABLE lp_runs;
|
||||||
|
-- Seeded category/groups are left in place (datasets may reference them).
|
||||||
Reference in New Issue
Block a user