feat: Add SLD endpoints, add date_to to vector_with_kato
This commit is contained in:
@@ -432,6 +432,54 @@ paths:
|
|||||||
"400": { $ref: "#/components/responses/BadRequest" }
|
"400": { $ref: "#/components/responses/BadRequest" }
|
||||||
"404": { $ref: "#/components/responses/NotFound" }
|
"404": { $ref: "#/components/responses/NotFound" }
|
||||||
|
|
||||||
|
/datasets/{id}/sld:
|
||||||
|
parameters:
|
||||||
|
- $ref: "#/components/parameters/IdParam"
|
||||||
|
post:
|
||||||
|
tags: [Datasets]
|
||||||
|
summary: Upload (or replace) a dataset's SLD style file
|
||||||
|
description: |
|
||||||
|
Multipart upload of an SLD (Styled Layer Descriptor) style file,
|
||||||
|
mirroring `POST /datasets`. Applies to every `file_type`. Uploading
|
||||||
|
again replaces the previously stored SLD.
|
||||||
|
|
||||||
|
Allowed extensions: `.sld`, `.xml`.
|
||||||
|
requestBody:
|
||||||
|
required: true
|
||||||
|
content:
|
||||||
|
multipart/form-data:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
required: [file]
|
||||||
|
properties:
|
||||||
|
file:
|
||||||
|
type: string
|
||||||
|
format: binary
|
||||||
|
description: The SLD file.
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: SLD stored; the updated dataset
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: "#/components/schemas/Dataset" }
|
||||||
|
"400": { $ref: "#/components/responses/BadRequest" }
|
||||||
|
"404": { $ref: "#/components/responses/NotFound" }
|
||||||
|
"422": { $ref: "#/components/responses/ValidationError" }
|
||||||
|
get:
|
||||||
|
tags: [Datasets]
|
||||||
|
summary: Download a dataset's SLD style file
|
||||||
|
description: Returns 404 if no SLD has been uploaded for this dataset yet.
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: The SLD file stream
|
||||||
|
content:
|
||||||
|
application/xml:
|
||||||
|
schema:
|
||||||
|
type: string
|
||||||
|
format: binary
|
||||||
|
"400": { $ref: "#/components/responses/BadRequest" }
|
||||||
|
"404": { $ref: "#/components/responses/NotFound" }
|
||||||
|
|
||||||
/datasets/{id}/mapping:
|
/datasets/{id}/mapping:
|
||||||
parameters:
|
parameters:
|
||||||
- $ref: "#/components/parameters/IdParam"
|
- $ref: "#/components/parameters/IdParam"
|
||||||
@@ -443,6 +491,10 @@ paths:
|
|||||||
`ready`, to re-map). The KATO column and every mapped column must be among
|
`ready`, to re-map). The KATO column and every mapped column must be among
|
||||||
the dataset's detected `attribute_columns`. Moves the dataset to
|
the dataset's detected `attribute_columns`. Moves the dataset to
|
||||||
`extracting`; observations are produced asynchronously.
|
`extracting`; observations are produced asynchronously.
|
||||||
|
|
||||||
|
Each year column maps to either a single point in time (`date` only,
|
||||||
|
the original behavior) or a period (`date` and `date_to`, inclusive).
|
||||||
|
`date_to`, when present, must not be before `date`.
|
||||||
requestBody:
|
requestBody:
|
||||||
required: true
|
required: true
|
||||||
content:
|
content:
|
||||||
@@ -672,6 +724,16 @@ components:
|
|||||||
date:
|
date:
|
||||||
type: string
|
type: string
|
||||||
format: date
|
format: date
|
||||||
|
description: >-
|
||||||
|
Point in time, or the start of the period when `date_to` is also
|
||||||
|
set.
|
||||||
|
date_to:
|
||||||
|
type: [string, "null"]
|
||||||
|
format: date
|
||||||
|
description: >-
|
||||||
|
Optional period end (inclusive), must not be before `date`. Omitted
|
||||||
|
or null means a single point in time (the original, pre-period
|
||||||
|
behavior).
|
||||||
|
|
||||||
GeoJSONGeometry:
|
GeoJSONGeometry:
|
||||||
type: [object, "null"]
|
type: [object, "null"]
|
||||||
@@ -699,6 +761,12 @@ components:
|
|||||||
date (YYYY-MM-DD). For the single-feature case (dataset has its own
|
date (YYYY-MM-DD). For the single-feature case (dataset has its own
|
||||||
geometry) it holds only the observations, keyed by KATO code, each
|
geometry) it holds only the observations, keyed by KATO code, each
|
||||||
mapping to its date->value pairs.
|
mapping to its date->value pairs.
|
||||||
|
|
||||||
|
For a `vector_with_kato` dataset mapped with one or more period
|
||||||
|
year columns (`date_to` set), a sibling `periods` object is also
|
||||||
|
present, mapping each period's start date (the same key used in
|
||||||
|
`data`) to its end date. `periods` is omitted entirely when the
|
||||||
|
dataset has no period mappings.
|
||||||
additionalProperties: true
|
additionalProperties: true
|
||||||
|
|
||||||
GeoJSONFeatureCollection:
|
GeoJSONFeatureCollection:
|
||||||
@@ -769,6 +837,13 @@ components:
|
|||||||
cog_storage_key:
|
cog_storage_key:
|
||||||
type: [string, "null"]
|
type: [string, "null"]
|
||||||
description: Set for rasters once the Cloud-Optimized GeoTIFF is produced.
|
description: Set for rasters once the Cloud-Optimized GeoTIFF is produced.
|
||||||
|
sld_storage_key:
|
||||||
|
type: [string, "null"]
|
||||||
|
description: Set once an SLD style file has been uploaded (any file_type).
|
||||||
|
sld_filename:
|
||||||
|
type: [string, "null"]
|
||||||
|
sld_content_type:
|
||||||
|
type: [string, "null"]
|
||||||
file_type:
|
file_type:
|
||||||
$ref: "#/components/schemas/FileType"
|
$ref: "#/components/schemas/FileType"
|
||||||
size_bytes:
|
size_bytes:
|
||||||
@@ -844,6 +919,12 @@ components:
|
|||||||
date:
|
date:
|
||||||
type: string
|
type: string
|
||||||
format: date
|
format: date
|
||||||
|
date_to:
|
||||||
|
type: [string, "null"]
|
||||||
|
format: date
|
||||||
|
description: >-
|
||||||
|
Period end (inclusive), carried over from the year mapping's
|
||||||
|
`date_to`. Null for a point-in-time observation.
|
||||||
value:
|
value:
|
||||||
type: [number, "null"]
|
type: [number, "null"]
|
||||||
description: Numeric cell value, or null when non-numeric/empty.
|
description: Numeric cell value, or null when non-numeric/empty.
|
||||||
|
|||||||
@@ -74,15 +74,19 @@ type DatasetFilter struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Observation is a single unpivoted value from a dataset's attribute table,
|
// Observation is a single unpivoted value from a dataset's attribute table,
|
||||||
// keyed by KATO code and date. Exactly one of Value / ValueText is typically
|
// keyed by KATO code and date (optionally a date..DateTo period). Exactly one
|
||||||
// set (numeric vs non-numeric cell); both may be nil for an empty cell.
|
// of Value / ValueText is typically set (numeric vs non-numeric cell); both
|
||||||
|
// may be nil for an empty cell.
|
||||||
type Observation struct {
|
type Observation struct {
|
||||||
ID uuid.UUID `json:"id"`
|
ID uuid.UUID `json:"id"`
|
||||||
DatasetID uuid.UUID `json:"dataset_id"`
|
DatasetID uuid.UUID `json:"dataset_id"`
|
||||||
KatoCode string `json:"kato_code"`
|
KatoCode string `json:"kato_code"`
|
||||||
Date string `json:"date"`
|
Date string `json:"date"`
|
||||||
Value *float64 `json:"value"`
|
// DateTo is the period's end date. Nil for a point-in-time observation
|
||||||
ValueText *string `json:"value_text"`
|
// (pre-existing behavior).
|
||||||
|
DateTo *string `json:"date_to,omitempty"`
|
||||||
|
Value *float64 `json:"value"`
|
||||||
|
ValueText *string `json:"value_text"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// allowedExtensions lists the accepted lowercase file extensions (including the
|
// allowedExtensions lists the accepted lowercase file extensions (including the
|
||||||
@@ -134,10 +138,30 @@ func ValidateFileContent(ext string, head []byte) error {
|
|||||||
if b, ok := firstMeaningfulByte(head); !ok || b != '{' {
|
if b, ok := firstMeaningfulByte(head); !ok || b != '{' {
|
||||||
return fmt.Errorf("file is not valid GeoJSON")
|
return fmt.Errorf("file is not valid GeoJSON")
|
||||||
}
|
}
|
||||||
|
case ".sld", ".xml":
|
||||||
|
// SLD is XML; a lightweight sniff for the opening tag.
|
||||||
|
if b, ok := firstMeaningfulByte(head); !ok || b != '<' {
|
||||||
|
return fmt.Errorf("file is not valid XML/SLD")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AllowedSLDExtensions lists the accepted lowercase extensions (including the
|
||||||
|
// dot) for a dataset's SLD (Styled Layer Descriptor) style file.
|
||||||
|
var AllowedSLDExtensions = []string{".sld", ".xml"}
|
||||||
|
|
||||||
|
// ExtensionAllowedForSLD reports whether ext (lowercase, with dot) is a valid
|
||||||
|
// SLD file extension.
|
||||||
|
func ExtensionAllowedForSLD(ext string) bool {
|
||||||
|
for _, e := range AllowedSLDExtensions {
|
||||||
|
if e == ext {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// firstMeaningfulByte returns the first non-whitespace byte after an optional
|
// firstMeaningfulByte returns the first non-whitespace byte after an optional
|
||||||
// UTF-8 BOM.
|
// UTF-8 BOM.
|
||||||
func firstMeaningfulByte(head []byte) (byte, bool) {
|
func firstMeaningfulByte(head []byte) (byte, bool) {
|
||||||
@@ -160,11 +184,15 @@ type AttributeColumn struct {
|
|||||||
Samples []string `json:"samples,omitempty"`
|
Samples []string `json:"samples,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// YearColumn maps an attribute column to the date it represents,
|
// YearColumn maps an attribute column to the date (or date period) it
|
||||||
// e.g. {"column": "F_2023", "date": "2023-01-01"}.
|
// represents, e.g. {"column": "F_2023", "date": "2023-01-01"} for a single
|
||||||
|
// point in time, or {"column": "F_2020_2023", "date": "2020-01-01", "date_to":
|
||||||
|
// "2023-12-31"} for a period. DateTo is optional; when nil the mapping is a
|
||||||
|
// single point in time, preserving the original (pre-period) behavior.
|
||||||
type YearColumn struct {
|
type YearColumn struct {
|
||||||
Column string `json:"column"`
|
Column string `json:"column"`
|
||||||
Date string `json:"date"`
|
Date string `json:"date"`
|
||||||
|
DateTo *string `json:"date_to,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DatasetSummary is the lightweight view of a dataset used in list responses.
|
// DatasetSummary is the lightweight view of a dataset used in list responses.
|
||||||
@@ -196,6 +224,11 @@ type Dataset struct {
|
|||||||
FileType FileType `json:"file_type"`
|
FileType FileType `json:"file_type"`
|
||||||
SizeBytes int64 `json:"size_bytes"`
|
SizeBytes int64 `json:"size_bytes"`
|
||||||
ContentType string `json:"content_type"`
|
ContentType string `json:"content_type"`
|
||||||
|
// SldStorageKey points to the dataset's SLD (Styled Layer Descriptor) style
|
||||||
|
// file, if one has been uploaded. Nullable; applies to every file type.
|
||||||
|
SldStorageKey *string `json:"sld_storage_key"`
|
||||||
|
SldFilename *string `json:"sld_filename"`
|
||||||
|
SldContentType *string `json:"sld_content_type"`
|
||||||
// Properties holds tabular data extracted from the file (e.g. a shapefile's
|
// Properties holds tabular data extracted from the file (e.g. a shapefile's
|
||||||
// attribute table). Nullable; not exposed in dataset responses.
|
// attribute table). Nullable; not exposed in dataset responses.
|
||||||
Properties json.RawMessage `json:"-"`
|
Properties json.RawMessage `json:"-"`
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ const datasetColumns = `id, category_id, name, description, unit, filename, stor
|
|||||||
CASE WHEN file_type = 'raster' AND geometry IS NOT NULL
|
CASE WHEN file_type = 'raster' AND geometry IS NOT NULL
|
||||||
THEN ARRAY[ST_XMin(geometry), ST_YMin(geometry), ST_XMax(geometry), ST_YMax(geometry)]
|
THEN ARRAY[ST_XMin(geometry), ST_YMin(geometry), ST_XMax(geometry), ST_YMax(geometry)]
|
||||||
ELSE NULL END AS bbox,
|
ELSE NULL END AS bbox,
|
||||||
geojson,
|
geojson, sld_storage_key, sld_filename, sld_content_type,
|
||||||
created_at, updated_at`
|
created_at, updated_at`
|
||||||
|
|
||||||
func scanDataset(row pgx.Row) (domain.Dataset, error) {
|
func scanDataset(row pgx.Row) (domain.Dataset, error) {
|
||||||
@@ -41,7 +41,8 @@ func scanDataset(row pgx.Row) (domain.Dataset, error) {
|
|||||||
&d.Filename, &d.StorageKey, &d.CogStorageKey, &d.FileType, &d.SizeBytes, &d.ContentType,
|
&d.Filename, &d.StorageKey, &d.CogStorageKey, &d.FileType, &d.SizeBytes, &d.ContentType,
|
||||||
&d.Properties, &d.Meta, &d.Automated, &d.Status,
|
&d.Properties, &d.Meta, &d.Automated, &d.Status,
|
||||||
&d.AttributeColumns, &d.KatoColumn, &d.YearColumns, &d.ParseError,
|
&d.AttributeColumns, &d.KatoColumn, &d.YearColumns, &d.ParseError,
|
||||||
&d.Geometry, &d.BBox, &d.GeoJSON, &d.CreatedAt, &d.UpdatedAt,
|
&d.Geometry, &d.BBox, &d.GeoJSON, &d.SldStorageKey, &d.SldFilename, &d.SldContentType,
|
||||||
|
&d.CreatedAt, &d.UpdatedAt,
|
||||||
)
|
)
|
||||||
return d, err
|
return d, err
|
||||||
}
|
}
|
||||||
@@ -228,6 +229,20 @@ func (r *DatasetRepository) SetGeoJSON(ctx context.Context, id uuid.UUID, geojso
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetSLD stores the dataset's SLD (Styled Layer Descriptor) style file
|
||||||
|
// metadata, replacing any previous one, and returns the updated row.
|
||||||
|
func (r *DatasetRepository) SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error) {
|
||||||
|
row := r.pool.QueryRow(ctx,
|
||||||
|
`UPDATE datasets
|
||||||
|
SET sld_storage_key = $2, sld_filename = $3, sld_content_type = $4, updated_at = now()
|
||||||
|
WHERE id = $1
|
||||||
|
RETURNING `+datasetColumns,
|
||||||
|
id, key, filename, contentType,
|
||||||
|
)
|
||||||
|
out, err := scanDataset(row)
|
||||||
|
return out, mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
// ClearGeometryAndGeoJSON nulls out a dataset's stored geometry and persisted
|
// ClearGeometryAndGeoJSON nulls out a dataset's stored geometry and persisted
|
||||||
// GeoJSON. Used before re-enqueuing a dataset's processing job so a stale
|
// GeoJSON. Used before re-enqueuing a dataset's processing job so a stale
|
||||||
// (possibly incorrect) geometry/GeoJSON is never served while the job is
|
// (possibly incorrect) geometry/GeoJSON is never served while the job is
|
||||||
@@ -265,11 +280,19 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("invalid observation date %q: %w", o.Date, err)
|
return fmt.Errorf("invalid observation date %q: %w", o.Date, err)
|
||||||
}
|
}
|
||||||
rows[i] = []any{datasetID, o.KatoCode, d, o.Value, o.ValueText}
|
var dateTo any
|
||||||
|
if o.DateTo != nil {
|
||||||
|
dt, err := time.Parse("2006-01-02", *o.DateTo)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid observation date_to %q: %w", *o.DateTo, err)
|
||||||
|
}
|
||||||
|
dateTo = dt
|
||||||
|
}
|
||||||
|
rows[i] = []any{datasetID, o.KatoCode, d, dateTo, o.Value, o.ValueText}
|
||||||
}
|
}
|
||||||
_, err := tx.CopyFrom(ctx,
|
_, err := tx.CopyFrom(ctx,
|
||||||
pgx.Identifier{"dataset_observations"},
|
pgx.Identifier{"dataset_observations"},
|
||||||
[]string{"dataset_id", "kato_code", "date", "value", "value_text"},
|
[]string{"dataset_id", "kato_code", "date", "date_to", "value", "value_text"},
|
||||||
pgx.CopyFromRows(rows),
|
pgx.CopyFromRows(rows),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -280,11 +303,11 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u
|
|||||||
return mapError(tx.Commit(ctx))
|
return mapError(tx.Commit(ctx))
|
||||||
}
|
}
|
||||||
|
|
||||||
const observationColumns = `id, dataset_id, kato_code, to_char(date, 'YYYY-MM-DD') AS date, value, value_text`
|
const observationColumns = `id, dataset_id, kato_code, to_char(date, 'YYYY-MM-DD') AS date, to_char(date_to, 'YYYY-MM-DD') AS date_to, value, value_text`
|
||||||
|
|
||||||
func scanObservation(row pgx.Row) (domain.Observation, error) {
|
func scanObservation(row pgx.Row) (domain.Observation, error) {
|
||||||
var o domain.Observation
|
var o domain.Observation
|
||||||
err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.Date, &o.Value, &o.ValueText)
|
err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.Date, &o.DateTo, &o.Value, &o.ValueText)
|
||||||
return o, err
|
return o, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ var (
|
|||||||
// ErrCOGNotReady is returned when a raster's Cloud-Optimized GeoTIFF has not
|
// ErrCOGNotReady is returned when a raster's Cloud-Optimized GeoTIFF has not
|
||||||
// been produced yet.
|
// been produced yet.
|
||||||
ErrCOGNotReady = errors.New("cog is not ready")
|
ErrCOGNotReady = errors.New("cog is not ready")
|
||||||
|
// ErrSLDNotFound is returned when a dataset has no SLD style file uploaded.
|
||||||
|
ErrSLDNotFound = fmt.Errorf("%w: dataset has no sld", domain.ErrNotFound)
|
||||||
)
|
)
|
||||||
|
|
||||||
// DatasetRepository is the persistence behaviour DatasetService needs.
|
// DatasetRepository is the persistence behaviour DatasetService needs.
|
||||||
@@ -47,6 +49,7 @@ type DatasetRepository interface {
|
|||||||
SetGeoJSON(ctx context.Context, id uuid.UUID, geojson []byte) error
|
SetGeoJSON(ctx context.Context, id uuid.UUID, geojson []byte) error
|
||||||
ClearGeometryAndGeoJSON(ctx context.Context, id uuid.UUID) error
|
ClearGeometryAndGeoJSON(ctx context.Context, id uuid.UUID) error
|
||||||
SaveMapping(ctx context.Context, id uuid.UUID, katoColumn string, years []domain.YearColumn) (domain.Dataset, error)
|
SaveMapping(ctx context.Context, id uuid.UUID, katoColumn string, years []domain.YearColumn) (domain.Dataset, error)
|
||||||
|
SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error)
|
||||||
ReplaceObservations(ctx context.Context, datasetID uuid.UUID, obs []domain.Observation) error
|
ReplaceObservations(ctx context.Context, datasetID uuid.UUID, obs []domain.Observation) error
|
||||||
ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error)
|
ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error)
|
||||||
CountObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string) (int, error)
|
CountObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string) (int, error)
|
||||||
@@ -235,6 +238,82 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat
|
|||||||
return dataset, nil
|
return dataset, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UploadSLDInput carries an SLD (Styled Layer Descriptor) style file to attach
|
||||||
|
// to an existing dataset.
|
||||||
|
type UploadSLDInput struct {
|
||||||
|
Filename string
|
||||||
|
ContentType string
|
||||||
|
Size int64
|
||||||
|
Reader io.Reader
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadSLD validates and stores an SLD style file for a dataset, replacing
|
||||||
|
// any previously stored one. Applies to every file_type.
|
||||||
|
func (s *DatasetService) UploadSLD(ctx context.Context, id uuid.UUID, in UploadSLDInput) (domain.Dataset, error) {
|
||||||
|
dataset, err := s.repo.GetByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Dataset{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
ext := strings.ToLower(filepath.Ext(in.Filename))
|
||||||
|
if !domain.ExtensionAllowedForSLD(ext) {
|
||||||
|
return domain.Dataset{}, fmt.Errorf("%w: sld file must have one of these extensions: %s",
|
||||||
|
domain.ErrValidation, strings.Join(domain.AllowedSLDExtensions, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
head := make([]byte, 512)
|
||||||
|
n, err := io.ReadFull(in.Reader, head)
|
||||||
|
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||||
|
return domain.Dataset{}, fmt.Errorf("read sld upload: %w", err)
|
||||||
|
}
|
||||||
|
head = head[:n]
|
||||||
|
if err := domain.ValidateFileContent(ext, head); err != nil {
|
||||||
|
return domain.Dataset{}, fmt.Errorf("%w: %s", domain.ErrValidation, err)
|
||||||
|
}
|
||||||
|
content := io.MultiReader(bytes.NewReader(head), in.Reader)
|
||||||
|
|
||||||
|
key := deriveSLDKey(dataset.StorageKey, in.Filename)
|
||||||
|
if err := s.store.Put(ctx, key, content, in.Size, in.ContentType); err != nil {
|
||||||
|
return domain.Dataset{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
updated, err := s.repo.SetSLD(ctx, id, key, in.Filename, in.ContentType)
|
||||||
|
if err != nil {
|
||||||
|
// Compensate: the row was not updated, so the object would be orphaned.
|
||||||
|
_ = s.store.Remove(ctx, key)
|
||||||
|
return domain.Dataset{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if dataset.SldStorageKey != nil && *dataset.SldStorageKey != key {
|
||||||
|
_ = s.store.Remove(ctx, *dataset.SldStorageKey) // best-effort: drop the replaced file
|
||||||
|
}
|
||||||
|
return updated, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadSLD returns the dataset metadata and a reader for its stored SLD
|
||||||
|
// style file. Fails with ErrSLDNotFound when none has been uploaded. The
|
||||||
|
// caller must close the reader.
|
||||||
|
func (s *DatasetService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain.Dataset, io.ReadCloser, error) {
|
||||||
|
dataset, err := s.repo.GetByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Dataset{}, nil, err
|
||||||
|
}
|
||||||
|
if dataset.SldStorageKey == nil {
|
||||||
|
return domain.Dataset{}, nil, ErrSLDNotFound
|
||||||
|
}
|
||||||
|
obj, err := s.store.Get(ctx, *dataset.SldStorageKey)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Dataset{}, nil, err
|
||||||
|
}
|
||||||
|
return dataset, obj, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deriveSLDKey places a dataset's SLD style file alongside the original under
|
||||||
|
// an sld/ prefix, named after the uploaded SLD file.
|
||||||
|
func deriveSLDKey(storageKey, filename string) string {
|
||||||
|
return path.Join(path.Dir(storageKey), "sld", filename)
|
||||||
|
}
|
||||||
|
|
||||||
// enqueueProcessing schedules the appropriate async job for a dataset based on
|
// enqueueProcessing schedules the appropriate async job for a dataset based on
|
||||||
// its file type: vector_with_kato is parsed for column selection, plain vector
|
// its file type: vector_with_kato is parsed for column selection, plain vector
|
||||||
// has its attribute table extracted into properties, and raster is converted to
|
// has its attribute table extracted into properties, and raster is converted to
|
||||||
@@ -603,9 +682,19 @@ func (s *DatasetService) SaveMapping(ctx context.Context, id uuid.UUID, in Mappi
|
|||||||
if _, ok := known[yc.Column]; !ok {
|
if _, ok := known[yc.Column]; !ok {
|
||||||
return domain.Dataset{}, fmt.Errorf("%w: year column %q is not among the detected columns", domain.ErrValidation, yc.Column)
|
return domain.Dataset{}, fmt.Errorf("%w: year column %q is not among the detected columns", domain.ErrValidation, yc.Column)
|
||||||
}
|
}
|
||||||
if _, err := time.Parse("2006-01-02", yc.Date); err != nil {
|
from, err := time.Parse("2006-01-02", yc.Date)
|
||||||
|
if err != nil {
|
||||||
return domain.Dataset{}, fmt.Errorf("%w: invalid date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.Date, yc.Column)
|
return domain.Dataset{}, fmt.Errorf("%w: invalid date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.Date, yc.Column)
|
||||||
}
|
}
|
||||||
|
if yc.DateTo != nil {
|
||||||
|
to, err := time.Parse("2006-01-02", *yc.DateTo)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Dataset{}, fmt.Errorf("%w: invalid date_to %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, *yc.DateTo, yc.Column)
|
||||||
|
}
|
||||||
|
if to.Before(from) {
|
||||||
|
return domain.Dataset{}, fmt.Errorf("%w: date_to %q is before date %q for column %q", domain.ErrValidation, *yc.DateTo, yc.Date, yc.Column)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
dataset, err = s.repo.SaveMapping(ctx, id, in.KatoColumn, in.YearColumns)
|
dataset, err = s.repo.SaveMapping(ctx, id, in.KatoColumn, in.YearColumns)
|
||||||
@@ -659,8 +748,9 @@ func (s *DatasetService) Extract(ctx context.Context, id uuid.UUID) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// buildObservations unpivots rows into observations. Rows without a KATO code
|
// buildObservations unpivots rows into observations. Rows without a KATO code
|
||||||
// are skipped; duplicate (kato, date) pairs keep the last value. Numeric cells
|
// are skipped; duplicate (kato, date, date_to) tuples keep the last value.
|
||||||
// populate Value, others ValueText.
|
// Numeric cells populate Value, others ValueText. A year column's DateTo (nil
|
||||||
|
// for a point-in-time mapping) carries through to the observation unchanged.
|
||||||
func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.YearColumn, rows []map[string]string) []domain.Observation {
|
func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.YearColumn, rows []map[string]string) []domain.Observation {
|
||||||
obs := make([]domain.Observation, 0, len(rows)*len(years))
|
obs := make([]domain.Observation, 0, len(rows)*len(years))
|
||||||
index := make(map[string]int)
|
index := make(map[string]int)
|
||||||
@@ -671,7 +761,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
for _, yc := range years {
|
for _, yc := range years {
|
||||||
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date}
|
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date, DateTo: yc.DateTo}
|
||||||
if raw := strings.TrimSpace(row[yc.Column]); raw != "" {
|
if raw := strings.TrimSpace(row[yc.Column]); raw != "" {
|
||||||
if f, err := strconv.ParseFloat(raw, 64); err == nil {
|
if f, err := strconv.ParseFloat(raw, 64); err == nil {
|
||||||
o.Value = &f
|
o.Value = &f
|
||||||
@@ -679,7 +769,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
|
|||||||
o.ValueText = &raw
|
o.ValueText = &raw
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
key := kato + "\x00" + yc.Date
|
key := kato + "\x00" + yc.Date + "\x00" + dateToKey(yc.DateTo)
|
||||||
if i, ok := index[key]; ok {
|
if i, ok := index[key]; ok {
|
||||||
obs[i] = o
|
obs[i] = o
|
||||||
} else {
|
} else {
|
||||||
@@ -691,6 +781,14 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
|
|||||||
return obs
|
return obs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// dateToKey normalizes an optional period end date for use in a dedup key.
|
||||||
|
func dateToKey(dateTo *string) string {
|
||||||
|
if dateTo == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return *dateTo
|
||||||
|
}
|
||||||
|
|
||||||
// ObservationPage is a page of observations with pagination metadata.
|
// ObservationPage is a page of observations with pagination metadata.
|
||||||
type ObservationPage struct {
|
type ObservationPage struct {
|
||||||
Items []domain.Observation
|
Items []domain.Observation
|
||||||
@@ -789,7 +887,11 @@ func (s *DatasetService) generateGeoJSON(ctx context.Context, id uuid.UUID) erro
|
|||||||
// joins the districts table on KATO code: one Feature is emitted per KATO, its
|
// joins the districts table on KATO code: one Feature is emitted per KATO, its
|
||||||
// boundary taken from the districts table and its observation values nested
|
// boundary taken from the districts table and its observation values nested
|
||||||
// under a `data` object (keyed by date) alongside `kato` and `name`. KATO codes
|
// under a `data` object (keyed by date) alongside `kato` and `name`. KATO codes
|
||||||
// with no matching district are skipped.
|
// with no matching district are skipped. When any observation carries a period
|
||||||
|
// (DateTo set), its end date is additionally exposed under a `periods` object
|
||||||
|
// (keyed by the same start date used in `data`); `periods` is omitted entirely
|
||||||
|
// when every observation is point-in-time, so a dataset mapped without periods
|
||||||
|
// produces byte-identical output to before periods existed.
|
||||||
func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.RawMessage, error) {
|
func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.RawMessage, error) {
|
||||||
dataset, err := s.repo.GetByID(ctx, id)
|
dataset, err := s.repo.GetByID(ctx, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -816,8 +918,8 @@ func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.R
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
grouped, order := groupObservationsByKato(obs)
|
grouped, periods, order := groupObservationsByKato(obs)
|
||||||
features, err := s.districtFeatures(ctx, grouped, order)
|
features, err := s.districtFeatures(ctx, grouped, periods, order)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -850,9 +952,12 @@ func (s *DatasetService) loadGeoJSONDataset(ctx context.Context, id uuid.UUID) (
|
|||||||
|
|
||||||
// districtFeatures builds one Feature per KATO from the districts table, with
|
// districtFeatures builds one Feature per KATO from the districts table, with
|
||||||
// the grouped observation values (keyed by date) nested under a `data` object
|
// the grouped observation values (keyed by date) nested under a `data` object
|
||||||
// alongside `kato` and `name` in each Feature's properties. KATO codes with no
|
// alongside `kato` and `name` in each Feature's properties. When periods holds
|
||||||
|
// an entry for the KATO (at least one observation had a period), its date->
|
||||||
|
// date_to map is additionally nested under a `periods` object; a KATO with no
|
||||||
|
// period observations gets no `periods` key at all. KATO codes with no
|
||||||
// matching district are skipped. order drives the deterministic feature order.
|
// matching district are skipped. order drives the deterministic feature order.
|
||||||
func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, order []string) ([]domain.Feature, error) {
|
func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, periods map[string]map[string]string, order []string) ([]domain.Feature, error) {
|
||||||
districts, err := s.repo.DistrictGeometriesByKato(ctx, order)
|
districts, err := s.repo.DistrictGeometriesByKato(ctx, order)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -863,10 +968,14 @@ func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[strin
|
|||||||
if !ok {
|
if !ok {
|
||||||
continue // skip KATO codes with no district boundary
|
continue // skip KATO codes with no district boundary
|
||||||
}
|
}
|
||||||
|
props := map[string]any{"kato": kato, "name": dist.Name, "data": grouped[kato]}
|
||||||
|
if p, ok := periods[kato]; ok {
|
||||||
|
props["periods"] = p
|
||||||
|
}
|
||||||
features = append(features, domain.Feature{
|
features = append(features, domain.Feature{
|
||||||
Type: domain.GeoJSONFeature,
|
Type: domain.GeoJSONFeature,
|
||||||
Geometry: dist.Geometry,
|
Geometry: dist.Geometry,
|
||||||
Properties: map[string]any{"kato": kato, "name": dist.Name, "data": grouped[kato]},
|
Properties: props,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return features, nil
|
return features, nil
|
||||||
@@ -909,11 +1018,15 @@ func vectorFeatureProperties(raw json.RawMessage) map[string]any {
|
|||||||
|
|
||||||
// groupObservationsByKato groups observations by KATO code into date->value
|
// groupObservationsByKato groups observations by KATO code into date->value
|
||||||
// maps. The value is the numeric Value when present, otherwise ValueText,
|
// maps. The value is the numeric Value when present, otherwise ValueText,
|
||||||
// otherwise nil (an empty cell). order lists KATO codes in first-seen order so
|
// otherwise nil (an empty cell). It also groups, per KATO, a date->date_to map
|
||||||
// the resulting feature order is deterministic.
|
// for observations that carry a period (DateTo set); a KATO with no period
|
||||||
func groupObservationsByKato(obs []domain.Observation) (map[string]map[string]any, []string) {
|
// observations gets no entry in periods at all, so callers can tell "no
|
||||||
grouped := make(map[string]map[string]any)
|
// periods" apart from "empty periods". order lists KATO codes in first-seen
|
||||||
order := make([]string, 0)
|
// order so the resulting feature order is deterministic.
|
||||||
|
func groupObservationsByKato(obs []domain.Observation) (grouped map[string]map[string]any, periods map[string]map[string]string, order []string) {
|
||||||
|
grouped = make(map[string]map[string]any)
|
||||||
|
periods = make(map[string]map[string]string)
|
||||||
|
order = make([]string, 0)
|
||||||
for _, o := range obs {
|
for _, o := range obs {
|
||||||
values, ok := grouped[o.KatoCode]
|
values, ok := grouped[o.KatoCode]
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -922,8 +1035,17 @@ func groupObservationsByKato(obs []domain.Observation) (map[string]map[string]an
|
|||||||
order = append(order, o.KatoCode)
|
order = append(order, o.KatoCode)
|
||||||
}
|
}
|
||||||
values[o.Date] = observationValue(o)
|
values[o.Date] = observationValue(o)
|
||||||
|
|
||||||
|
if o.DateTo != nil {
|
||||||
|
p, ok := periods[o.KatoCode]
|
||||||
|
if !ok {
|
||||||
|
p = make(map[string]string)
|
||||||
|
periods[o.KatoCode] = p
|
||||||
|
}
|
||||||
|
p[o.Date] = *o.DateTo
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return grouped, order
|
return grouped, periods, order
|
||||||
}
|
}
|
||||||
|
|
||||||
// observationValue returns the typed cell value: numeric Value, else ValueText,
|
// observationValue returns the typed cell value: numeric Value, else ValueText,
|
||||||
|
|||||||
@@ -137,6 +137,18 @@ func (r *stubDatasetRepo) SaveMapping(_ context.Context, id uuid.UUID, kato stri
|
|||||||
return d, nil
|
return d, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *stubDatasetRepo) SetSLD(_ context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error) {
|
||||||
|
d, ok := r.store[id]
|
||||||
|
if !ok {
|
||||||
|
return domain.Dataset{}, domain.ErrNotFound
|
||||||
|
}
|
||||||
|
d.SldStorageKey = &key
|
||||||
|
d.SldFilename = &filename
|
||||||
|
d.SldContentType = &contentType
|
||||||
|
r.store[id] = d
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *stubDatasetRepo) MarkReady(_ context.Context, id uuid.UUID, geometry []byte) error {
|
func (r *stubDatasetRepo) MarkReady(_ context.Context, id uuid.UUID, geometry []byte) error {
|
||||||
d, ok := r.store[id]
|
d, ok := r.store[id]
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -776,6 +788,39 @@ func TestDatasetService_SaveMapping(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("bad date_to", func(t *testing.T) {
|
||||||
|
_, svc := newSvc()
|
||||||
|
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-01-01", DateTo: strPtr("2023")}}})
|
||||||
|
if !errors.Is(err, domain.ErrValidation) {
|
||||||
|
t.Fatalf("want ErrValidation, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("date_to before date", func(t *testing.T) {
|
||||||
|
_, svc := newSvc()
|
||||||
|
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-12-31", DateTo: strPtr("2023-01-01")}}})
|
||||||
|
if !errors.Is(err, domain.ErrValidation) {
|
||||||
|
t.Fatalf("want ErrValidation, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("period mapping succeeds", func(t *testing.T) {
|
||||||
|
repo, svc := newSvc()
|
||||||
|
got, err := svc.SaveMapping(ctx, id, MappingInput{
|
||||||
|
KatoColumn: "като",
|
||||||
|
YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2020-01-01", DateTo: strPtr("2023-12-31")}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(got.YearColumns) != 1 || got.YearColumns[0].DateTo == nil || *got.YearColumns[0].DateTo != "2023-12-31" {
|
||||||
|
t.Fatalf("period not saved: %+v", got.YearColumns)
|
||||||
|
}
|
||||||
|
if len(repo.store[id].YearColumns) != 1 {
|
||||||
|
t.Fatalf("period not persisted: %+v", repo.store[id].YearColumns)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("no year columns", func(t *testing.T) {
|
t.Run("no year columns", func(t *testing.T) {
|
||||||
_, svc := newSvc()
|
_, svc := newSvc()
|
||||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като"})
|
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като"})
|
||||||
@@ -845,6 +890,45 @@ func TestBuildObservations(t *testing.T) {
|
|||||||
if o := byKey["751010000|2025-01-01"]; o.ValueText == nil || *o.ValueText != "n/a" {
|
if o := byKey["751010000|2025-01-01"]; o.ValueText == nil || *o.ValueText != "n/a" {
|
||||||
t.Errorf("non-numeric cell not stored as value_text: %+v", o)
|
t.Errorf("non-numeric cell not stored as value_text: %+v", o)
|
||||||
}
|
}
|
||||||
|
for _, o := range obs {
|
||||||
|
if o.DateTo != nil {
|
||||||
|
t.Fatalf("point-in-time mapping must not set DateTo, got %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildObservations_Period(t *testing.T) {
|
||||||
|
id := uuid.New()
|
||||||
|
years := []domain.YearColumn{
|
||||||
|
{Column: "F_2020_2023", Date: "2020-01-01", DateTo: strPtr("2023-12-31")}, // period
|
||||||
|
{Column: "F_2024", Date: "2024-01-01"}, // point-in-time, unaffected
|
||||||
|
}
|
||||||
|
rows := []map[string]string{
|
||||||
|
{"като": "751010000", "F_2020_2023": "100", "F_2024": "200"},
|
||||||
|
}
|
||||||
|
|
||||||
|
obs := buildObservations(id, "като", years, rows)
|
||||||
|
if len(obs) != 2 {
|
||||||
|
t.Fatalf("want 2 observations, got %d", len(obs))
|
||||||
|
}
|
||||||
|
|
||||||
|
byDate := map[string]domain.Observation{}
|
||||||
|
for _, o := range obs {
|
||||||
|
byDate[o.Date] = o
|
||||||
|
}
|
||||||
|
|
||||||
|
period := byDate["2020-01-01"]
|
||||||
|
if period.DateTo == nil || *period.DateTo != "2023-12-31" {
|
||||||
|
t.Fatalf("period observation missing date_to: %+v", period)
|
||||||
|
}
|
||||||
|
if period.Value == nil || *period.Value != 100 {
|
||||||
|
t.Fatalf("period observation value wrong: %+v", period)
|
||||||
|
}
|
||||||
|
|
||||||
|
point := byDate["2024-01-01"]
|
||||||
|
if point.DateTo != nil {
|
||||||
|
t.Fatalf("point-in-time observation must have nil date_to (backward compat), got %+v", point)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDatasetService_Extract(t *testing.T) {
|
func TestDatasetService_Extract(t *testing.T) {
|
||||||
@@ -959,6 +1043,68 @@ func TestDatasetService_GeoJSON_DistrictJoin(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDatasetService_GeoJSON_Period(t *testing.T) {
|
||||||
|
id := uuid.New()
|
||||||
|
repo := newStubDatasetRepo()
|
||||||
|
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeVectorWithKato, Status: domain.DatasetStatusReady}
|
||||||
|
|
||||||
|
period, point := 100.0, 200.0
|
||||||
|
repo.observations[id] = []domain.Observation{
|
||||||
|
{KatoCode: "710000000", Date: "2020-01-01", DateTo: strPtr("2023-12-31"), Value: &period},
|
||||||
|
{KatoCode: "710000000", Date: "2024-01-01", Value: &point}, // point-in-time, no period
|
||||||
|
}
|
||||||
|
repo.districts["710000000"] = domain.District{
|
||||||
|
Kato: "710000000", Name: "Astana",
|
||||||
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`),
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := newDatasetService(repo, &stubStore{}, true)
|
||||||
|
fc := mustGeoJSON(t, svc, id)
|
||||||
|
if len(fc.Features) != 1 {
|
||||||
|
t.Fatalf("want 1 feature, got %d", len(fc.Features))
|
||||||
|
}
|
||||||
|
f := fc.Features[0]
|
||||||
|
|
||||||
|
data, ok := f.Properties["data"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("observations not nested under data: %+v", f.Properties)
|
||||||
|
}
|
||||||
|
if data["2020-01-01"] != 100.0 || data["2024-01-01"] != 200.0 {
|
||||||
|
t.Fatalf("unexpected data: %+v", data)
|
||||||
|
}
|
||||||
|
|
||||||
|
periods, ok := f.Properties["periods"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("periods not exposed as a map: %+v", f.Properties)
|
||||||
|
}
|
||||||
|
if len(periods) != 1 || periods["2020-01-01"] != "2023-12-31" {
|
||||||
|
t.Fatalf("period end date not exposed: %+v", periods)
|
||||||
|
}
|
||||||
|
if _, has := periods["2024-01-01"]; has {
|
||||||
|
t.Fatalf("point-in-time date must not appear in periods: %+v", periods)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetService_GeoJSON_NoPeriods_OmitsPeriodsKey(t *testing.T) {
|
||||||
|
// Regression: a dataset mapped without any period must produce output
|
||||||
|
// byte-identical to before periods existed - no "periods" key at all.
|
||||||
|
id := uuid.New()
|
||||||
|
repo := newStubDatasetRepo()
|
||||||
|
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeVectorWithKato, Status: domain.DatasetStatusReady}
|
||||||
|
v := 100.0
|
||||||
|
repo.observations[id] = []domain.Observation{{KatoCode: "710000000", Date: "2020-01-01", Value: &v}}
|
||||||
|
repo.districts["710000000"] = domain.District{
|
||||||
|
Kato: "710000000", Name: "Astana",
|
||||||
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`),
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := newDatasetService(repo, &stubStore{}, true)
|
||||||
|
fc := mustGeoJSON(t, svc, id)
|
||||||
|
if _, has := fc.Features[0].Properties["periods"]; has {
|
||||||
|
t.Fatalf("periods key must be absent when no observation has a period: %+v", fc.Features[0].Properties)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDatasetService_GeoJSON_IgnoresDatasetGeometry(t *testing.T) {
|
func TestDatasetService_GeoJSON_IgnoresDatasetGeometry(t *testing.T) {
|
||||||
id := uuid.New()
|
id := uuid.New()
|
||||||
repo := newStubDatasetRepo()
|
repo := newStubDatasetRepo()
|
||||||
@@ -1294,6 +1440,125 @@ func TestDatasetService_ReprocessAll_FiltersByFileType(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validSLDUpload() UploadSLDInput {
|
||||||
|
body := `<?xml version="1.0"?><StyledLayerDescriptor></StyledLayerDescriptor>`
|
||||||
|
return UploadSLDInput{
|
||||||
|
Filename: "style.sld",
|
||||||
|
ContentType: "application/xml",
|
||||||
|
Size: int64(len(body)),
|
||||||
|
Reader: strings.NewReader(body),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetService_UploadSLD_Success(t *testing.T) {
|
||||||
|
id := uuid.New()
|
||||||
|
repo := newStubDatasetRepo()
|
||||||
|
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeRaster, StorageKey: "uid/dem.tif"}
|
||||||
|
store := &stubStore{}
|
||||||
|
svc := newDatasetService(repo, store, true)
|
||||||
|
|
||||||
|
got, err := svc.UploadSLD(context.Background(), id, validSLDUpload())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if got.SldStorageKey == nil || *got.SldStorageKey != "uid/sld/style.sld" {
|
||||||
|
t.Fatalf("unexpected sld storage key: %v", got.SldStorageKey)
|
||||||
|
}
|
||||||
|
if got.SldFilename == nil || *got.SldFilename != "style.sld" {
|
||||||
|
t.Fatalf("sld filename not stored: %v", got.SldFilename)
|
||||||
|
}
|
||||||
|
if len(store.put) != 1 || store.put[0] != "uid/sld/style.sld" {
|
||||||
|
t.Fatalf("sld not uploaded to store: %v", store.put)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetService_UploadSLD_ReplacesPrevious(t *testing.T) {
|
||||||
|
id := uuid.New()
|
||||||
|
repo := newStubDatasetRepo()
|
||||||
|
oldKey := "uid/sld/old.sld"
|
||||||
|
repo.store[id] = domain.Dataset{
|
||||||
|
ID: id, FileType: domain.FileTypeVector, StorageKey: "uid/data.geojson",
|
||||||
|
SldStorageKey: &oldKey,
|
||||||
|
}
|
||||||
|
store := &stubStore{}
|
||||||
|
svc := newDatasetService(repo, store, true)
|
||||||
|
|
||||||
|
in := validSLDUpload()
|
||||||
|
in.Filename = "new.sld"
|
||||||
|
got, err := svc.UploadSLD(context.Background(), id, in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if got.SldStorageKey == nil || *got.SldStorageKey != "uid/sld/new.sld" {
|
||||||
|
t.Fatalf("unexpected sld storage key: %v", got.SldStorageKey)
|
||||||
|
}
|
||||||
|
if len(store.removed) != 1 || store.removed[0] != oldKey {
|
||||||
|
t.Fatalf("old sld object not removed: %v", store.removed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetService_UploadSLD_RejectsBadExtension(t *testing.T) {
|
||||||
|
id := uuid.New()
|
||||||
|
repo := newStubDatasetRepo()
|
||||||
|
repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"}
|
||||||
|
store := &stubStore{}
|
||||||
|
svc := newDatasetService(repo, store, true)
|
||||||
|
|
||||||
|
in := validSLDUpload()
|
||||||
|
in.Filename = "style.zip"
|
||||||
|
_, err := svc.UploadSLD(context.Background(), id, in)
|
||||||
|
if !errors.Is(err, domain.ErrValidation) {
|
||||||
|
t.Fatalf("want ErrValidation, got %v", err)
|
||||||
|
}
|
||||||
|
if len(store.put) != 0 {
|
||||||
|
t.Fatalf("nothing should be uploaded on validation failure, got %v", store.put)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetService_UploadSLD_RejectsMismatchedContent(t *testing.T) {
|
||||||
|
id := uuid.New()
|
||||||
|
repo := newStubDatasetRepo()
|
||||||
|
repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"}
|
||||||
|
store := &stubStore{}
|
||||||
|
svc := newDatasetService(repo, store, true)
|
||||||
|
|
||||||
|
in := validSLDUpload()
|
||||||
|
in.Reader = strings.NewReader(`{"not":"xml"}`)
|
||||||
|
_, err := svc.UploadSLD(context.Background(), id, in)
|
||||||
|
if !errors.Is(err, domain.ErrValidation) {
|
||||||
|
t.Fatalf("want ErrValidation, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetService_DownloadSLD(t *testing.T) {
|
||||||
|
id := uuid.New()
|
||||||
|
repo := newStubDatasetRepo()
|
||||||
|
repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"}
|
||||||
|
store := &stubStore{}
|
||||||
|
svc := newDatasetService(repo, store, true)
|
||||||
|
|
||||||
|
t.Run("not found before upload", func(t *testing.T) {
|
||||||
|
_, _, err := svc.DownloadSLD(context.Background(), id)
|
||||||
|
if !errors.Is(err, ErrSLDNotFound) {
|
||||||
|
t.Fatalf("want ErrSLDNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("streams after upload", func(t *testing.T) {
|
||||||
|
if _, err := svc.UploadSLD(context.Background(), id, validSLDUpload()); err != nil {
|
||||||
|
t.Fatalf("upload: %v", err)
|
||||||
|
}
|
||||||
|
_, obj, err := svc.DownloadSLD(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
defer obj.Close()
|
||||||
|
if _, err := io.ReadAll(obj); err != nil {
|
||||||
|
t.Fatalf("read: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestDatasetService_Delete_RemovesObject(t *testing.T) {
|
func TestDatasetService_Delete_RemovesObject(t *testing.T) {
|
||||||
repo := newStubDatasetRepo()
|
repo := newStubDatasetRepo()
|
||||||
id := uuid.New()
|
id := uuid.New()
|
||||||
|
|||||||
@@ -43,6 +43,8 @@ func (h *DatasetHandler) Register(r chi.Router) {
|
|||||||
r.Get("/{id}.cog", h.cog)
|
r.Get("/{id}.cog", h.cog)
|
||||||
r.Get("/{id}/status", h.status)
|
r.Get("/{id}/status", h.status)
|
||||||
r.Get("/{id}/download", h.download)
|
r.Get("/{id}/download", h.download)
|
||||||
|
r.Post("/{id}/sld", h.uploadSLD)
|
||||||
|
r.Get("/{id}/sld", h.downloadSLD)
|
||||||
r.Post("/{id}/mapping", h.mapping)
|
r.Post("/{id}/mapping", h.mapping)
|
||||||
r.Get("/{id}/observations", h.observations)
|
r.Get("/{id}/observations", h.observations)
|
||||||
r.Delete("/{id}", h.delete)
|
r.Delete("/{id}", h.delete)
|
||||||
@@ -78,8 +80,9 @@ func (h *DatasetHandler) status(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type yearColumnInput struct {
|
type yearColumnInput struct {
|
||||||
Column string `json:"column" validate:"required"`
|
Column string `json:"column" validate:"required"`
|
||||||
Date string `json:"date" validate:"required,datetime=2006-01-02"`
|
Date string `json:"date" validate:"required,datetime=2006-01-02"`
|
||||||
|
DateTo *string `json:"date_to" validate:"omitempty,datetime=2006-01-02"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type mappingRequest struct {
|
type mappingRequest struct {
|
||||||
@@ -106,7 +109,7 @@ func (h *DatasetHandler) mapping(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
in := service.MappingInput{KatoColumn: req.KatoColumn}
|
in := service.MappingInput{KatoColumn: req.KatoColumn}
|
||||||
for _, yc := range req.YearColumns {
|
for _, yc := range req.YearColumns {
|
||||||
in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, Date: yc.Date})
|
in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, Date: yc.Date, DateTo: yc.DateTo})
|
||||||
}
|
}
|
||||||
|
|
||||||
dataset, err := h.svc.SaveMapping(r.Context(), id, in)
|
dataset, err := h.svc.SaveMapping(r.Context(), id, in)
|
||||||
@@ -533,6 +536,74 @@ func (h *DatasetHandler) download(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// uploadSLD stores (or replaces) a dataset's SLD (Styled Layer Descriptor)
|
||||||
|
// style file. Multipart upload, mirroring POST /datasets. Applies to every
|
||||||
|
// file_type.
|
||||||
|
func (h *DatasetHandler) uploadSLD(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
file, header, err := r.FormFile("file")
|
||||||
|
if err != nil {
|
||||||
|
httputil.WriteError(w, http.StatusBadRequest, "file is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
contentType := header.Header.Get("Content-Type")
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/xml"
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset, err := h.svc.UploadSLD(r.Context(), id, service.UploadSLDInput{
|
||||||
|
Filename: header.Filename,
|
||||||
|
ContentType: contentType,
|
||||||
|
Size: header.Size,
|
||||||
|
Reader: file,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, dataset)
|
||||||
|
}
|
||||||
|
|
||||||
|
// downloadSLD streams a dataset's stored SLD style file. Returns 404 if none
|
||||||
|
// has been uploaded yet.
|
||||||
|
func (h *DatasetHandler) downloadSLD(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
dataset, obj, err := h.svc.DownloadSLD(r.Context(), id)
|
||||||
|
if err != nil {
|
||||||
|
respondDomainError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer obj.Close()
|
||||||
|
|
||||||
|
contentType := "application/xml"
|
||||||
|
if dataset.SldContentType != nil && *dataset.SldContentType != "" {
|
||||||
|
contentType = *dataset.SldContentType
|
||||||
|
}
|
||||||
|
filename := "style.sld"
|
||||||
|
if dataset.SldFilename != nil && *dataset.SldFilename != "" {
|
||||||
|
filename = *dataset.SldFilename
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", contentType)
|
||||||
|
w.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`)
|
||||||
|
if _, err := io.Copy(w, obj); err != nil {
|
||||||
|
// Headers are already sent; nothing useful to return to the client.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (h *DatasetHandler) delete(w http.ResponseWriter, r *http.Request) {
|
func (h *DatasetHandler) delete(w http.ResponseWriter, r *http.Request) {
|
||||||
id, ok := parseUUIDParam(w, r, "id")
|
id, ok := parseUUIDParam(w, r, "id")
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|||||||
12
migrations/00008_add_sld_to_datasets.sql
Normal file
12
migrations/00008_add_sld_to_datasets.sql
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
-- +goose Up
|
||||||
|
-- Optional SLD (Styled Layer Descriptor) style file for a dataset. Applies to
|
||||||
|
-- every file_type; a dataset may have at most one (uploading again replaces
|
||||||
|
-- it).
|
||||||
|
ALTER TABLE datasets ADD COLUMN sld_storage_key TEXT;
|
||||||
|
ALTER TABLE datasets ADD COLUMN sld_filename VARCHAR(255);
|
||||||
|
ALTER TABLE datasets ADD COLUMN sld_content_type VARCHAR(255);
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
ALTER TABLE datasets DROP COLUMN sld_storage_key;
|
||||||
|
ALTER TABLE datasets DROP COLUMN sld_filename;
|
||||||
|
ALTER TABLE datasets DROP COLUMN sld_content_type;
|
||||||
15
migrations/00009_add_period_to_dataset_observations.sql
Normal file
15
migrations/00009_add_period_to_dataset_observations.sql
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
-- +goose Up
|
||||||
|
-- Optional period end date, so a year_column mapping can represent a date
|
||||||
|
-- range (date..date_to) instead of a single point in time. NULL preserves the
|
||||||
|
-- pre-existing point-in-time behavior; the unique index treats a NULL date_to
|
||||||
|
-- as equal to date itself so point observations keep deduplicating exactly as
|
||||||
|
-- before.
|
||||||
|
ALTER TABLE dataset_observations ADD COLUMN date_to DATE;
|
||||||
|
|
||||||
|
DROP INDEX uq_dataset_observations;
|
||||||
|
CREATE UNIQUE INDEX uq_dataset_observations ON dataset_observations (dataset_id, kato_code, date, COALESCE(date_to, date));
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
DROP INDEX uq_dataset_observations;
|
||||||
|
ALTER TABLE dataset_observations DROP COLUMN date_to;
|
||||||
|
CREATE UNIQUE INDEX uq_dataset_observations ON dataset_observations (dataset_id, kato_code, date);
|
||||||
Reference in New Issue
Block a user