feat: Add SLD endpoints, add date_to to vector_with_kato

This commit is contained in:
2026-08-01 22:09:59 +05:00
parent 00a0cd1b8e
commit 1e0844ca82
8 changed files with 656 additions and 34 deletions

View File

@@ -432,6 +432,54 @@ paths:
"400": { $ref: "#/components/responses/BadRequest" } "400": { $ref: "#/components/responses/BadRequest" }
"404": { $ref: "#/components/responses/NotFound" } "404": { $ref: "#/components/responses/NotFound" }
/datasets/{id}/sld:
parameters:
- $ref: "#/components/parameters/IdParam"
post:
tags: [Datasets]
summary: Upload (or replace) a dataset's SLD style file
description: |
Multipart upload of an SLD (Styled Layer Descriptor) style file,
mirroring `POST /datasets`. Applies to every `file_type`. Uploading
again replaces the previously stored SLD.
Allowed extensions: `.sld`, `.xml`.
requestBody:
required: true
content:
multipart/form-data:
schema:
type: object
required: [file]
properties:
file:
type: string
format: binary
description: The SLD file.
responses:
"200":
description: SLD stored; the updated dataset
content:
application/json:
schema: { $ref: "#/components/schemas/Dataset" }
"400": { $ref: "#/components/responses/BadRequest" }
"404": { $ref: "#/components/responses/NotFound" }
"422": { $ref: "#/components/responses/ValidationError" }
get:
tags: [Datasets]
summary: Download a dataset's SLD style file
description: Returns 404 if no SLD has been uploaded for this dataset yet.
responses:
"200":
description: The SLD file stream
content:
application/xml:
schema:
type: string
format: binary
"400": { $ref: "#/components/responses/BadRequest" }
"404": { $ref: "#/components/responses/NotFound" }
/datasets/{id}/mapping: /datasets/{id}/mapping:
parameters: parameters:
- $ref: "#/components/parameters/IdParam" - $ref: "#/components/parameters/IdParam"
@@ -443,6 +491,10 @@ paths:
`ready`, to re-map). The KATO column and every mapped column must be among `ready`, to re-map). The KATO column and every mapped column must be among
the dataset's detected `attribute_columns`. Moves the dataset to the dataset's detected `attribute_columns`. Moves the dataset to
`extracting`; observations are produced asynchronously. `extracting`; observations are produced asynchronously.
Each year column maps to either a single point in time (`date` only,
the original behavior) or a period (`date` and `date_to`, inclusive).
`date_to`, when present, must not be before `date`.
requestBody: requestBody:
required: true required: true
content: content:
@@ -672,6 +724,16 @@ components:
date: date:
type: string type: string
format: date format: date
description: >-
Point in time, or the start of the period when `date_to` is also
set.
date_to:
type: [string, "null"]
format: date
description: >-
Optional period end (inclusive), must not be before `date`. Omitted
or null means a single point in time (the original, pre-period
behavior).
GeoJSONGeometry: GeoJSONGeometry:
type: [object, "null"] type: [object, "null"]
@@ -699,6 +761,12 @@ components:
date (YYYY-MM-DD). For the single-feature case (dataset has its own date (YYYY-MM-DD). For the single-feature case (dataset has its own
geometry) it holds only the observations, keyed by KATO code, each geometry) it holds only the observations, keyed by KATO code, each
mapping to its date->value pairs. mapping to its date->value pairs.
For a `vector_with_kato` dataset mapped with one or more period
year columns (`date_to` set), a sibling `periods` object is also
present, mapping each period's start date (the same key used in
`data`) to its end date. `periods` is omitted entirely when the
dataset has no period mappings.
additionalProperties: true additionalProperties: true
GeoJSONFeatureCollection: GeoJSONFeatureCollection:
@@ -769,6 +837,13 @@ components:
cog_storage_key: cog_storage_key:
type: [string, "null"] type: [string, "null"]
description: Set for rasters once the Cloud-Optimized GeoTIFF is produced. description: Set for rasters once the Cloud-Optimized GeoTIFF is produced.
sld_storage_key:
type: [string, "null"]
description: Set once an SLD style file has been uploaded (any file_type).
sld_filename:
type: [string, "null"]
sld_content_type:
type: [string, "null"]
file_type: file_type:
$ref: "#/components/schemas/FileType" $ref: "#/components/schemas/FileType"
size_bytes: size_bytes:
@@ -844,6 +919,12 @@ components:
date: date:
type: string type: string
format: date format: date
date_to:
type: [string, "null"]
format: date
description: >-
Period end (inclusive), carried over from the year mapping's
`date_to`. Null for a point-in-time observation.
value: value:
type: [number, "null"] type: [number, "null"]
description: Numeric cell value, or null when non-numeric/empty. description: Numeric cell value, or null when non-numeric/empty.

View File

@@ -74,15 +74,19 @@ type DatasetFilter struct {
} }
// Observation is a single unpivoted value from a dataset's attribute table, // Observation is a single unpivoted value from a dataset's attribute table,
// keyed by KATO code and date. Exactly one of Value / ValueText is typically // keyed by KATO code and date (optionally a date..DateTo period). Exactly one
// set (numeric vs non-numeric cell); both may be nil for an empty cell. // of Value / ValueText is typically set (numeric vs non-numeric cell); both
// may be nil for an empty cell.
type Observation struct { type Observation struct {
ID uuid.UUID `json:"id"` ID uuid.UUID `json:"id"`
DatasetID uuid.UUID `json:"dataset_id"` DatasetID uuid.UUID `json:"dataset_id"`
KatoCode string `json:"kato_code"` KatoCode string `json:"kato_code"`
Date string `json:"date"` Date string `json:"date"`
Value *float64 `json:"value"` // DateTo is the period's end date. Nil for a point-in-time observation
ValueText *string `json:"value_text"` // (pre-existing behavior).
DateTo *string `json:"date_to,omitempty"`
Value *float64 `json:"value"`
ValueText *string `json:"value_text"`
} }
// allowedExtensions lists the accepted lowercase file extensions (including the // allowedExtensions lists the accepted lowercase file extensions (including the
@@ -134,10 +138,30 @@ func ValidateFileContent(ext string, head []byte) error {
if b, ok := firstMeaningfulByte(head); !ok || b != '{' { if b, ok := firstMeaningfulByte(head); !ok || b != '{' {
return fmt.Errorf("file is not valid GeoJSON") return fmt.Errorf("file is not valid GeoJSON")
} }
case ".sld", ".xml":
// SLD is XML; a lightweight sniff for the opening tag.
if b, ok := firstMeaningfulByte(head); !ok || b != '<' {
return fmt.Errorf("file is not valid XML/SLD")
}
} }
return nil return nil
} }
// AllowedSLDExtensions lists the accepted lowercase extensions (including the
// dot) for a dataset's SLD (Styled Layer Descriptor) style file.
var AllowedSLDExtensions = []string{".sld", ".xml"}
// ExtensionAllowedForSLD reports whether ext (lowercase, with dot) is a valid
// SLD file extension.
func ExtensionAllowedForSLD(ext string) bool {
for _, e := range AllowedSLDExtensions {
if e == ext {
return true
}
}
return false
}
// firstMeaningfulByte returns the first non-whitespace byte after an optional // firstMeaningfulByte returns the first non-whitespace byte after an optional
// UTF-8 BOM. // UTF-8 BOM.
func firstMeaningfulByte(head []byte) (byte, bool) { func firstMeaningfulByte(head []byte) (byte, bool) {
@@ -160,11 +184,15 @@ type AttributeColumn struct {
Samples []string `json:"samples,omitempty"` Samples []string `json:"samples,omitempty"`
} }
// YearColumn maps an attribute column to the date it represents, // YearColumn maps an attribute column to the date (or date period) it
// e.g. {"column": "F_2023", "date": "2023-01-01"}. // represents, e.g. {"column": "F_2023", "date": "2023-01-01"} for a single
// point in time, or {"column": "F_2020_2023", "date": "2020-01-01", "date_to":
// "2023-12-31"} for a period. DateTo is optional; when nil the mapping is a
// single point in time, preserving the original (pre-period) behavior.
type YearColumn struct { type YearColumn struct {
Column string `json:"column"` Column string `json:"column"`
Date string `json:"date"` Date string `json:"date"`
DateTo *string `json:"date_to,omitempty"`
} }
// DatasetSummary is the lightweight view of a dataset used in list responses. // DatasetSummary is the lightweight view of a dataset used in list responses.
@@ -196,6 +224,11 @@ type Dataset struct {
FileType FileType `json:"file_type"` FileType FileType `json:"file_type"`
SizeBytes int64 `json:"size_bytes"` SizeBytes int64 `json:"size_bytes"`
ContentType string `json:"content_type"` ContentType string `json:"content_type"`
// SldStorageKey points to the dataset's SLD (Styled Layer Descriptor) style
// file, if one has been uploaded. Nullable; applies to every file type.
SldStorageKey *string `json:"sld_storage_key"`
SldFilename *string `json:"sld_filename"`
SldContentType *string `json:"sld_content_type"`
// Properties holds tabular data extracted from the file (e.g. a shapefile's // Properties holds tabular data extracted from the file (e.g. a shapefile's
// attribute table). Nullable; not exposed in dataset responses. // attribute table). Nullable; not exposed in dataset responses.
Properties json.RawMessage `json:"-"` Properties json.RawMessage `json:"-"`

View File

@@ -31,7 +31,7 @@ const datasetColumns = `id, category_id, name, description, unit, filename, stor
CASE WHEN file_type = 'raster' AND geometry IS NOT NULL CASE WHEN file_type = 'raster' AND geometry IS NOT NULL
THEN ARRAY[ST_XMin(geometry), ST_YMin(geometry), ST_XMax(geometry), ST_YMax(geometry)] THEN ARRAY[ST_XMin(geometry), ST_YMin(geometry), ST_XMax(geometry), ST_YMax(geometry)]
ELSE NULL END AS bbox, ELSE NULL END AS bbox,
geojson, geojson, sld_storage_key, sld_filename, sld_content_type,
created_at, updated_at` created_at, updated_at`
func scanDataset(row pgx.Row) (domain.Dataset, error) { func scanDataset(row pgx.Row) (domain.Dataset, error) {
@@ -41,7 +41,8 @@ func scanDataset(row pgx.Row) (domain.Dataset, error) {
&d.Filename, &d.StorageKey, &d.CogStorageKey, &d.FileType, &d.SizeBytes, &d.ContentType, &d.Filename, &d.StorageKey, &d.CogStorageKey, &d.FileType, &d.SizeBytes, &d.ContentType,
&d.Properties, &d.Meta, &d.Automated, &d.Status, &d.Properties, &d.Meta, &d.Automated, &d.Status,
&d.AttributeColumns, &d.KatoColumn, &d.YearColumns, &d.ParseError, &d.AttributeColumns, &d.KatoColumn, &d.YearColumns, &d.ParseError,
&d.Geometry, &d.BBox, &d.GeoJSON, &d.CreatedAt, &d.UpdatedAt, &d.Geometry, &d.BBox, &d.GeoJSON, &d.SldStorageKey, &d.SldFilename, &d.SldContentType,
&d.CreatedAt, &d.UpdatedAt,
) )
return d, err return d, err
} }
@@ -228,6 +229,20 @@ func (r *DatasetRepository) SetGeoJSON(ctx context.Context, id uuid.UUID, geojso
return nil return nil
} }
// SetSLD stores the dataset's SLD (Styled Layer Descriptor) style file
// metadata, replacing any previous one, and returns the updated row.
func (r *DatasetRepository) SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error) {
row := r.pool.QueryRow(ctx,
`UPDATE datasets
SET sld_storage_key = $2, sld_filename = $3, sld_content_type = $4, updated_at = now()
WHERE id = $1
RETURNING `+datasetColumns,
id, key, filename, contentType,
)
out, err := scanDataset(row)
return out, mapError(err)
}
// ClearGeometryAndGeoJSON nulls out a dataset's stored geometry and persisted // ClearGeometryAndGeoJSON nulls out a dataset's stored geometry and persisted
// GeoJSON. Used before re-enqueuing a dataset's processing job so a stale // GeoJSON. Used before re-enqueuing a dataset's processing job so a stale
// (possibly incorrect) geometry/GeoJSON is never served while the job is // (possibly incorrect) geometry/GeoJSON is never served while the job is
@@ -265,11 +280,19 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u
if err != nil { if err != nil {
return fmt.Errorf("invalid observation date %q: %w", o.Date, err) return fmt.Errorf("invalid observation date %q: %w", o.Date, err)
} }
rows[i] = []any{datasetID, o.KatoCode, d, o.Value, o.ValueText} var dateTo any
if o.DateTo != nil {
dt, err := time.Parse("2006-01-02", *o.DateTo)
if err != nil {
return fmt.Errorf("invalid observation date_to %q: %w", *o.DateTo, err)
}
dateTo = dt
}
rows[i] = []any{datasetID, o.KatoCode, d, dateTo, o.Value, o.ValueText}
} }
_, err := tx.CopyFrom(ctx, _, err := tx.CopyFrom(ctx,
pgx.Identifier{"dataset_observations"}, pgx.Identifier{"dataset_observations"},
[]string{"dataset_id", "kato_code", "date", "value", "value_text"}, []string{"dataset_id", "kato_code", "date", "date_to", "value", "value_text"},
pgx.CopyFromRows(rows), pgx.CopyFromRows(rows),
) )
if err != nil { if err != nil {
@@ -280,11 +303,11 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u
return mapError(tx.Commit(ctx)) return mapError(tx.Commit(ctx))
} }
const observationColumns = `id, dataset_id, kato_code, to_char(date, 'YYYY-MM-DD') AS date, value, value_text` const observationColumns = `id, dataset_id, kato_code, to_char(date, 'YYYY-MM-DD') AS date, to_char(date_to, 'YYYY-MM-DD') AS date_to, value, value_text`
func scanObservation(row pgx.Row) (domain.Observation, error) { func scanObservation(row pgx.Row) (domain.Observation, error) {
var o domain.Observation var o domain.Observation
err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.Date, &o.Value, &o.ValueText) err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.Date, &o.DateTo, &o.Value, &o.ValueText)
return o, err return o, err
} }

View File

@@ -29,6 +29,8 @@ var (
// ErrCOGNotReady is returned when a raster's Cloud-Optimized GeoTIFF has not // ErrCOGNotReady is returned when a raster's Cloud-Optimized GeoTIFF has not
// been produced yet. // been produced yet.
ErrCOGNotReady = errors.New("cog is not ready") ErrCOGNotReady = errors.New("cog is not ready")
// ErrSLDNotFound is returned when a dataset has no SLD style file uploaded.
ErrSLDNotFound = fmt.Errorf("%w: dataset has no sld", domain.ErrNotFound)
) )
// DatasetRepository is the persistence behaviour DatasetService needs. // DatasetRepository is the persistence behaviour DatasetService needs.
@@ -47,6 +49,7 @@ type DatasetRepository interface {
SetGeoJSON(ctx context.Context, id uuid.UUID, geojson []byte) error SetGeoJSON(ctx context.Context, id uuid.UUID, geojson []byte) error
ClearGeometryAndGeoJSON(ctx context.Context, id uuid.UUID) error ClearGeometryAndGeoJSON(ctx context.Context, id uuid.UUID) error
SaveMapping(ctx context.Context, id uuid.UUID, katoColumn string, years []domain.YearColumn) (domain.Dataset, error) SaveMapping(ctx context.Context, id uuid.UUID, katoColumn string, years []domain.YearColumn) (domain.Dataset, error)
SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error)
ReplaceObservations(ctx context.Context, datasetID uuid.UUID, obs []domain.Observation) error ReplaceObservations(ctx context.Context, datasetID uuid.UUID, obs []domain.Observation) error
ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error) ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error)
CountObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string) (int, error) CountObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string) (int, error)
@@ -235,6 +238,82 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat
return dataset, nil return dataset, nil
} }
// UploadSLDInput carries an SLD (Styled Layer Descriptor) style file to attach
// to an existing dataset.
type UploadSLDInput struct {
Filename string
ContentType string
Size int64
Reader io.Reader
}
// UploadSLD validates and stores an SLD style file for a dataset, replacing
// any previously stored one. Applies to every file_type.
func (s *DatasetService) UploadSLD(ctx context.Context, id uuid.UUID, in UploadSLDInput) (domain.Dataset, error) {
dataset, err := s.repo.GetByID(ctx, id)
if err != nil {
return domain.Dataset{}, err
}
ext := strings.ToLower(filepath.Ext(in.Filename))
if !domain.ExtensionAllowedForSLD(ext) {
return domain.Dataset{}, fmt.Errorf("%w: sld file must have one of these extensions: %s",
domain.ErrValidation, strings.Join(domain.AllowedSLDExtensions, ", "))
}
head := make([]byte, 512)
n, err := io.ReadFull(in.Reader, head)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return domain.Dataset{}, fmt.Errorf("read sld upload: %w", err)
}
head = head[:n]
if err := domain.ValidateFileContent(ext, head); err != nil {
return domain.Dataset{}, fmt.Errorf("%w: %s", domain.ErrValidation, err)
}
content := io.MultiReader(bytes.NewReader(head), in.Reader)
key := deriveSLDKey(dataset.StorageKey, in.Filename)
if err := s.store.Put(ctx, key, content, in.Size, in.ContentType); err != nil {
return domain.Dataset{}, err
}
updated, err := s.repo.SetSLD(ctx, id, key, in.Filename, in.ContentType)
if err != nil {
// Compensate: the row was not updated, so the object would be orphaned.
_ = s.store.Remove(ctx, key)
return domain.Dataset{}, err
}
if dataset.SldStorageKey != nil && *dataset.SldStorageKey != key {
_ = s.store.Remove(ctx, *dataset.SldStorageKey) // best-effort: drop the replaced file
}
return updated, nil
}
// DownloadSLD returns the dataset metadata and a reader for its stored SLD
// style file. Fails with ErrSLDNotFound when none has been uploaded. The
// caller must close the reader.
func (s *DatasetService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain.Dataset, io.ReadCloser, error) {
dataset, err := s.repo.GetByID(ctx, id)
if err != nil {
return domain.Dataset{}, nil, err
}
if dataset.SldStorageKey == nil {
return domain.Dataset{}, nil, ErrSLDNotFound
}
obj, err := s.store.Get(ctx, *dataset.SldStorageKey)
if err != nil {
return domain.Dataset{}, nil, err
}
return dataset, obj, nil
}
// deriveSLDKey places a dataset's SLD style file alongside the original under
// an sld/ prefix, named after the uploaded SLD file.
func deriveSLDKey(storageKey, filename string) string {
return path.Join(path.Dir(storageKey), "sld", filename)
}
// enqueueProcessing schedules the appropriate async job for a dataset based on // enqueueProcessing schedules the appropriate async job for a dataset based on
// its file type: vector_with_kato is parsed for column selection, plain vector // its file type: vector_with_kato is parsed for column selection, plain vector
// has its attribute table extracted into properties, and raster is converted to // has its attribute table extracted into properties, and raster is converted to
@@ -603,9 +682,19 @@ func (s *DatasetService) SaveMapping(ctx context.Context, id uuid.UUID, in Mappi
if _, ok := known[yc.Column]; !ok { if _, ok := known[yc.Column]; !ok {
return domain.Dataset{}, fmt.Errorf("%w: year column %q is not among the detected columns", domain.ErrValidation, yc.Column) return domain.Dataset{}, fmt.Errorf("%w: year column %q is not among the detected columns", domain.ErrValidation, yc.Column)
} }
if _, err := time.Parse("2006-01-02", yc.Date); err != nil { from, err := time.Parse("2006-01-02", yc.Date)
if err != nil {
return domain.Dataset{}, fmt.Errorf("%w: invalid date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.Date, yc.Column) return domain.Dataset{}, fmt.Errorf("%w: invalid date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.Date, yc.Column)
} }
if yc.DateTo != nil {
to, err := time.Parse("2006-01-02", *yc.DateTo)
if err != nil {
return domain.Dataset{}, fmt.Errorf("%w: invalid date_to %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, *yc.DateTo, yc.Column)
}
if to.Before(from) {
return domain.Dataset{}, fmt.Errorf("%w: date_to %q is before date %q for column %q", domain.ErrValidation, *yc.DateTo, yc.Date, yc.Column)
}
}
} }
dataset, err = s.repo.SaveMapping(ctx, id, in.KatoColumn, in.YearColumns) dataset, err = s.repo.SaveMapping(ctx, id, in.KatoColumn, in.YearColumns)
@@ -659,8 +748,9 @@ func (s *DatasetService) Extract(ctx context.Context, id uuid.UUID) error {
} }
// buildObservations unpivots rows into observations. Rows without a KATO code // buildObservations unpivots rows into observations. Rows without a KATO code
// are skipped; duplicate (kato, date) pairs keep the last value. Numeric cells // are skipped; duplicate (kato, date, date_to) tuples keep the last value.
// populate Value, others ValueText. // Numeric cells populate Value, others ValueText. A year column's DateTo (nil
// for a point-in-time mapping) carries through to the observation unchanged.
func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.YearColumn, rows []map[string]string) []domain.Observation { func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.YearColumn, rows []map[string]string) []domain.Observation {
obs := make([]domain.Observation, 0, len(rows)*len(years)) obs := make([]domain.Observation, 0, len(rows)*len(years))
index := make(map[string]int) index := make(map[string]int)
@@ -671,7 +761,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
continue continue
} }
for _, yc := range years { for _, yc := range years {
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date} o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date, DateTo: yc.DateTo}
if raw := strings.TrimSpace(row[yc.Column]); raw != "" { if raw := strings.TrimSpace(row[yc.Column]); raw != "" {
if f, err := strconv.ParseFloat(raw, 64); err == nil { if f, err := strconv.ParseFloat(raw, 64); err == nil {
o.Value = &f o.Value = &f
@@ -679,7 +769,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
o.ValueText = &raw o.ValueText = &raw
} }
} }
key := kato + "\x00" + yc.Date key := kato + "\x00" + yc.Date + "\x00" + dateToKey(yc.DateTo)
if i, ok := index[key]; ok { if i, ok := index[key]; ok {
obs[i] = o obs[i] = o
} else { } else {
@@ -691,6 +781,14 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
return obs return obs
} }
// dateToKey normalizes an optional period end date for use in a dedup key.
func dateToKey(dateTo *string) string {
if dateTo == nil {
return ""
}
return *dateTo
}
// ObservationPage is a page of observations with pagination metadata. // ObservationPage is a page of observations with pagination metadata.
type ObservationPage struct { type ObservationPage struct {
Items []domain.Observation Items []domain.Observation
@@ -789,7 +887,11 @@ func (s *DatasetService) generateGeoJSON(ctx context.Context, id uuid.UUID) erro
// joins the districts table on KATO code: one Feature is emitted per KATO, its // joins the districts table on KATO code: one Feature is emitted per KATO, its
// boundary taken from the districts table and its observation values nested // boundary taken from the districts table and its observation values nested
// under a `data` object (keyed by date) alongside `kato` and `name`. KATO codes // under a `data` object (keyed by date) alongside `kato` and `name`. KATO codes
// with no matching district are skipped. // with no matching district are skipped. When any observation carries a period
// (DateTo set), its end date is additionally exposed under a `periods` object
// (keyed by the same start date used in `data`); `periods` is omitted entirely
// when every observation is point-in-time, so a dataset mapped without periods
// produces byte-identical output to before periods existed.
func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.RawMessage, error) { func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.RawMessage, error) {
dataset, err := s.repo.GetByID(ctx, id) dataset, err := s.repo.GetByID(ctx, id)
if err != nil { if err != nil {
@@ -816,8 +918,8 @@ func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.R
if err != nil { if err != nil {
return nil, err return nil, err
} }
grouped, order := groupObservationsByKato(obs) grouped, periods, order := groupObservationsByKato(obs)
features, err := s.districtFeatures(ctx, grouped, order) features, err := s.districtFeatures(ctx, grouped, periods, order)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -850,9 +952,12 @@ func (s *DatasetService) loadGeoJSONDataset(ctx context.Context, id uuid.UUID) (
// districtFeatures builds one Feature per KATO from the districts table, with // districtFeatures builds one Feature per KATO from the districts table, with
// the grouped observation values (keyed by date) nested under a `data` object // the grouped observation values (keyed by date) nested under a `data` object
// alongside `kato` and `name` in each Feature's properties. KATO codes with no // alongside `kato` and `name` in each Feature's properties. When periods holds
// an entry for the KATO (at least one observation had a period), its date->
// date_to map is additionally nested under a `periods` object; a KATO with no
// period observations gets no `periods` key at all. KATO codes with no
// matching district are skipped. order drives the deterministic feature order. // matching district are skipped. order drives the deterministic feature order.
func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, order []string) ([]domain.Feature, error) { func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, periods map[string]map[string]string, order []string) ([]domain.Feature, error) {
districts, err := s.repo.DistrictGeometriesByKato(ctx, order) districts, err := s.repo.DistrictGeometriesByKato(ctx, order)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -863,10 +968,14 @@ func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[strin
if !ok { if !ok {
continue // skip KATO codes with no district boundary continue // skip KATO codes with no district boundary
} }
props := map[string]any{"kato": kato, "name": dist.Name, "data": grouped[kato]}
if p, ok := periods[kato]; ok {
props["periods"] = p
}
features = append(features, domain.Feature{ features = append(features, domain.Feature{
Type: domain.GeoJSONFeature, Type: domain.GeoJSONFeature,
Geometry: dist.Geometry, Geometry: dist.Geometry,
Properties: map[string]any{"kato": kato, "name": dist.Name, "data": grouped[kato]}, Properties: props,
}) })
} }
return features, nil return features, nil
@@ -909,11 +1018,15 @@ func vectorFeatureProperties(raw json.RawMessage) map[string]any {
// groupObservationsByKato groups observations by KATO code into date->value // groupObservationsByKato groups observations by KATO code into date->value
// maps. The value is the numeric Value when present, otherwise ValueText, // maps. The value is the numeric Value when present, otherwise ValueText,
// otherwise nil (an empty cell). order lists KATO codes in first-seen order so // otherwise nil (an empty cell). It also groups, per KATO, a date->date_to map
// the resulting feature order is deterministic. // for observations that carry a period (DateTo set); a KATO with no period
func groupObservationsByKato(obs []domain.Observation) (map[string]map[string]any, []string) { // observations gets no entry in periods at all, so callers can tell "no
grouped := make(map[string]map[string]any) // periods" apart from "empty periods". order lists KATO codes in first-seen
order := make([]string, 0) // order so the resulting feature order is deterministic.
func groupObservationsByKato(obs []domain.Observation) (grouped map[string]map[string]any, periods map[string]map[string]string, order []string) {
grouped = make(map[string]map[string]any)
periods = make(map[string]map[string]string)
order = make([]string, 0)
for _, o := range obs { for _, o := range obs {
values, ok := grouped[o.KatoCode] values, ok := grouped[o.KatoCode]
if !ok { if !ok {
@@ -922,8 +1035,17 @@ func groupObservationsByKato(obs []domain.Observation) (map[string]map[string]an
order = append(order, o.KatoCode) order = append(order, o.KatoCode)
} }
values[o.Date] = observationValue(o) values[o.Date] = observationValue(o)
if o.DateTo != nil {
p, ok := periods[o.KatoCode]
if !ok {
p = make(map[string]string)
periods[o.KatoCode] = p
}
p[o.Date] = *o.DateTo
}
} }
return grouped, order return grouped, periods, order
} }
// observationValue returns the typed cell value: numeric Value, else ValueText, // observationValue returns the typed cell value: numeric Value, else ValueText,

View File

@@ -137,6 +137,18 @@ func (r *stubDatasetRepo) SaveMapping(_ context.Context, id uuid.UUID, kato stri
return d, nil return d, nil
} }
func (r *stubDatasetRepo) SetSLD(_ context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error) {
d, ok := r.store[id]
if !ok {
return domain.Dataset{}, domain.ErrNotFound
}
d.SldStorageKey = &key
d.SldFilename = &filename
d.SldContentType = &contentType
r.store[id] = d
return d, nil
}
func (r *stubDatasetRepo) MarkReady(_ context.Context, id uuid.UUID, geometry []byte) error { func (r *stubDatasetRepo) MarkReady(_ context.Context, id uuid.UUID, geometry []byte) error {
d, ok := r.store[id] d, ok := r.store[id]
if !ok { if !ok {
@@ -776,6 +788,39 @@ func TestDatasetService_SaveMapping(t *testing.T) {
} }
}) })
t.Run("bad date_to", func(t *testing.T) {
_, svc := newSvc()
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-01-01", DateTo: strPtr("2023")}}})
if !errors.Is(err, domain.ErrValidation) {
t.Fatalf("want ErrValidation, got %v", err)
}
})
t.Run("date_to before date", func(t *testing.T) {
_, svc := newSvc()
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-12-31", DateTo: strPtr("2023-01-01")}}})
if !errors.Is(err, domain.ErrValidation) {
t.Fatalf("want ErrValidation, got %v", err)
}
})
t.Run("period mapping succeeds", func(t *testing.T) {
repo, svc := newSvc()
got, err := svc.SaveMapping(ctx, id, MappingInput{
KatoColumn: "като",
YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2020-01-01", DateTo: strPtr("2023-12-31")}},
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(got.YearColumns) != 1 || got.YearColumns[0].DateTo == nil || *got.YearColumns[0].DateTo != "2023-12-31" {
t.Fatalf("period not saved: %+v", got.YearColumns)
}
if len(repo.store[id].YearColumns) != 1 {
t.Fatalf("period not persisted: %+v", repo.store[id].YearColumns)
}
})
t.Run("no year columns", func(t *testing.T) { t.Run("no year columns", func(t *testing.T) {
_, svc := newSvc() _, svc := newSvc()
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като"}) _, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като"})
@@ -845,6 +890,45 @@ func TestBuildObservations(t *testing.T) {
if o := byKey["751010000|2025-01-01"]; o.ValueText == nil || *o.ValueText != "n/a" { if o := byKey["751010000|2025-01-01"]; o.ValueText == nil || *o.ValueText != "n/a" {
t.Errorf("non-numeric cell not stored as value_text: %+v", o) t.Errorf("non-numeric cell not stored as value_text: %+v", o)
} }
for _, o := range obs {
if o.DateTo != nil {
t.Fatalf("point-in-time mapping must not set DateTo, got %+v", o)
}
}
}
func TestBuildObservations_Period(t *testing.T) {
id := uuid.New()
years := []domain.YearColumn{
{Column: "F_2020_2023", Date: "2020-01-01", DateTo: strPtr("2023-12-31")}, // period
{Column: "F_2024", Date: "2024-01-01"}, // point-in-time, unaffected
}
rows := []map[string]string{
{"като": "751010000", "F_2020_2023": "100", "F_2024": "200"},
}
obs := buildObservations(id, "като", years, rows)
if len(obs) != 2 {
t.Fatalf("want 2 observations, got %d", len(obs))
}
byDate := map[string]domain.Observation{}
for _, o := range obs {
byDate[o.Date] = o
}
period := byDate["2020-01-01"]
if period.DateTo == nil || *period.DateTo != "2023-12-31" {
t.Fatalf("period observation missing date_to: %+v", period)
}
if period.Value == nil || *period.Value != 100 {
t.Fatalf("period observation value wrong: %+v", period)
}
point := byDate["2024-01-01"]
if point.DateTo != nil {
t.Fatalf("point-in-time observation must have nil date_to (backward compat), got %+v", point)
}
} }
func TestDatasetService_Extract(t *testing.T) { func TestDatasetService_Extract(t *testing.T) {
@@ -959,6 +1043,68 @@ func TestDatasetService_GeoJSON_DistrictJoin(t *testing.T) {
} }
} }
func TestDatasetService_GeoJSON_Period(t *testing.T) {
id := uuid.New()
repo := newStubDatasetRepo()
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeVectorWithKato, Status: domain.DatasetStatusReady}
period, point := 100.0, 200.0
repo.observations[id] = []domain.Observation{
{KatoCode: "710000000", Date: "2020-01-01", DateTo: strPtr("2023-12-31"), Value: &period},
{KatoCode: "710000000", Date: "2024-01-01", Value: &point}, // point-in-time, no period
}
repo.districts["710000000"] = domain.District{
Kato: "710000000", Name: "Astana",
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`),
}
svc := newDatasetService(repo, &stubStore{}, true)
fc := mustGeoJSON(t, svc, id)
if len(fc.Features) != 1 {
t.Fatalf("want 1 feature, got %d", len(fc.Features))
}
f := fc.Features[0]
data, ok := f.Properties["data"].(map[string]any)
if !ok {
t.Fatalf("observations not nested under data: %+v", f.Properties)
}
if data["2020-01-01"] != 100.0 || data["2024-01-01"] != 200.0 {
t.Fatalf("unexpected data: %+v", data)
}
periods, ok := f.Properties["periods"].(map[string]any)
if !ok {
t.Fatalf("periods not exposed as a map: %+v", f.Properties)
}
if len(periods) != 1 || periods["2020-01-01"] != "2023-12-31" {
t.Fatalf("period end date not exposed: %+v", periods)
}
if _, has := periods["2024-01-01"]; has {
t.Fatalf("point-in-time date must not appear in periods: %+v", periods)
}
}
func TestDatasetService_GeoJSON_NoPeriods_OmitsPeriodsKey(t *testing.T) {
// Regression: a dataset mapped without any period must produce output
// byte-identical to before periods existed - no "periods" key at all.
id := uuid.New()
repo := newStubDatasetRepo()
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeVectorWithKato, Status: domain.DatasetStatusReady}
v := 100.0
repo.observations[id] = []domain.Observation{{KatoCode: "710000000", Date: "2020-01-01", Value: &v}}
repo.districts["710000000"] = domain.District{
Kato: "710000000", Name: "Astana",
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`),
}
svc := newDatasetService(repo, &stubStore{}, true)
fc := mustGeoJSON(t, svc, id)
if _, has := fc.Features[0].Properties["periods"]; has {
t.Fatalf("periods key must be absent when no observation has a period: %+v", fc.Features[0].Properties)
}
}
func TestDatasetService_GeoJSON_IgnoresDatasetGeometry(t *testing.T) { func TestDatasetService_GeoJSON_IgnoresDatasetGeometry(t *testing.T) {
id := uuid.New() id := uuid.New()
repo := newStubDatasetRepo() repo := newStubDatasetRepo()
@@ -1294,6 +1440,125 @@ func TestDatasetService_ReprocessAll_FiltersByFileType(t *testing.T) {
} }
} }
func validSLDUpload() UploadSLDInput {
body := `<?xml version="1.0"?><StyledLayerDescriptor></StyledLayerDescriptor>`
return UploadSLDInput{
Filename: "style.sld",
ContentType: "application/xml",
Size: int64(len(body)),
Reader: strings.NewReader(body),
}
}
func TestDatasetService_UploadSLD_Success(t *testing.T) {
id := uuid.New()
repo := newStubDatasetRepo()
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeRaster, StorageKey: "uid/dem.tif"}
store := &stubStore{}
svc := newDatasetService(repo, store, true)
got, err := svc.UploadSLD(context.Background(), id, validSLDUpload())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got.SldStorageKey == nil || *got.SldStorageKey != "uid/sld/style.sld" {
t.Fatalf("unexpected sld storage key: %v", got.SldStorageKey)
}
if got.SldFilename == nil || *got.SldFilename != "style.sld" {
t.Fatalf("sld filename not stored: %v", got.SldFilename)
}
if len(store.put) != 1 || store.put[0] != "uid/sld/style.sld" {
t.Fatalf("sld not uploaded to store: %v", store.put)
}
}
func TestDatasetService_UploadSLD_ReplacesPrevious(t *testing.T) {
id := uuid.New()
repo := newStubDatasetRepo()
oldKey := "uid/sld/old.sld"
repo.store[id] = domain.Dataset{
ID: id, FileType: domain.FileTypeVector, StorageKey: "uid/data.geojson",
SldStorageKey: &oldKey,
}
store := &stubStore{}
svc := newDatasetService(repo, store, true)
in := validSLDUpload()
in.Filename = "new.sld"
got, err := svc.UploadSLD(context.Background(), id, in)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got.SldStorageKey == nil || *got.SldStorageKey != "uid/sld/new.sld" {
t.Fatalf("unexpected sld storage key: %v", got.SldStorageKey)
}
if len(store.removed) != 1 || store.removed[0] != oldKey {
t.Fatalf("old sld object not removed: %v", store.removed)
}
}
func TestDatasetService_UploadSLD_RejectsBadExtension(t *testing.T) {
id := uuid.New()
repo := newStubDatasetRepo()
repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"}
store := &stubStore{}
svc := newDatasetService(repo, store, true)
in := validSLDUpload()
in.Filename = "style.zip"
_, err := svc.UploadSLD(context.Background(), id, in)
if !errors.Is(err, domain.ErrValidation) {
t.Fatalf("want ErrValidation, got %v", err)
}
if len(store.put) != 0 {
t.Fatalf("nothing should be uploaded on validation failure, got %v", store.put)
}
}
func TestDatasetService_UploadSLD_RejectsMismatchedContent(t *testing.T) {
id := uuid.New()
repo := newStubDatasetRepo()
repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"}
store := &stubStore{}
svc := newDatasetService(repo, store, true)
in := validSLDUpload()
in.Reader = strings.NewReader(`{"not":"xml"}`)
_, err := svc.UploadSLD(context.Background(), id, in)
if !errors.Is(err, domain.ErrValidation) {
t.Fatalf("want ErrValidation, got %v", err)
}
}
func TestDatasetService_DownloadSLD(t *testing.T) {
id := uuid.New()
repo := newStubDatasetRepo()
repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"}
store := &stubStore{}
svc := newDatasetService(repo, store, true)
t.Run("not found before upload", func(t *testing.T) {
_, _, err := svc.DownloadSLD(context.Background(), id)
if !errors.Is(err, ErrSLDNotFound) {
t.Fatalf("want ErrSLDNotFound, got %v", err)
}
})
t.Run("streams after upload", func(t *testing.T) {
if _, err := svc.UploadSLD(context.Background(), id, validSLDUpload()); err != nil {
t.Fatalf("upload: %v", err)
}
_, obj, err := svc.DownloadSLD(context.Background(), id)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer obj.Close()
if _, err := io.ReadAll(obj); err != nil {
t.Fatalf("read: %v", err)
}
})
}
func TestDatasetService_Delete_RemovesObject(t *testing.T) { func TestDatasetService_Delete_RemovesObject(t *testing.T) {
repo := newStubDatasetRepo() repo := newStubDatasetRepo()
id := uuid.New() id := uuid.New()

View File

@@ -43,6 +43,8 @@ func (h *DatasetHandler) Register(r chi.Router) {
r.Get("/{id}.cog", h.cog) r.Get("/{id}.cog", h.cog)
r.Get("/{id}/status", h.status) r.Get("/{id}/status", h.status)
r.Get("/{id}/download", h.download) r.Get("/{id}/download", h.download)
r.Post("/{id}/sld", h.uploadSLD)
r.Get("/{id}/sld", h.downloadSLD)
r.Post("/{id}/mapping", h.mapping) r.Post("/{id}/mapping", h.mapping)
r.Get("/{id}/observations", h.observations) r.Get("/{id}/observations", h.observations)
r.Delete("/{id}", h.delete) r.Delete("/{id}", h.delete)
@@ -78,8 +80,9 @@ func (h *DatasetHandler) status(w http.ResponseWriter, r *http.Request) {
} }
type yearColumnInput struct { type yearColumnInput struct {
Column string `json:"column" validate:"required"` Column string `json:"column" validate:"required"`
Date string `json:"date" validate:"required,datetime=2006-01-02"` Date string `json:"date" validate:"required,datetime=2006-01-02"`
DateTo *string `json:"date_to" validate:"omitempty,datetime=2006-01-02"`
} }
type mappingRequest struct { type mappingRequest struct {
@@ -106,7 +109,7 @@ func (h *DatasetHandler) mapping(w http.ResponseWriter, r *http.Request) {
in := service.MappingInput{KatoColumn: req.KatoColumn} in := service.MappingInput{KatoColumn: req.KatoColumn}
for _, yc := range req.YearColumns { for _, yc := range req.YearColumns {
in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, Date: yc.Date}) in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, Date: yc.Date, DateTo: yc.DateTo})
} }
dataset, err := h.svc.SaveMapping(r.Context(), id, in) dataset, err := h.svc.SaveMapping(r.Context(), id, in)
@@ -533,6 +536,74 @@ func (h *DatasetHandler) download(w http.ResponseWriter, r *http.Request) {
} }
} }
// uploadSLD stores (or replaces) a dataset's SLD (Styled Layer Descriptor)
// style file. Multipart upload, mirroring POST /datasets. Applies to every
// file_type.
func (h *DatasetHandler) uploadSLD(w http.ResponseWriter, r *http.Request) {
id, ok := parseUUIDParam(w, r, "id")
if !ok {
return
}
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
return
}
file, header, err := r.FormFile("file")
if err != nil {
httputil.WriteError(w, http.StatusBadRequest, "file is required")
return
}
defer file.Close()
contentType := header.Header.Get("Content-Type")
if contentType == "" {
contentType = "application/xml"
}
dataset, err := h.svc.UploadSLD(r.Context(), id, service.UploadSLDInput{
Filename: header.Filename,
ContentType: contentType,
Size: header.Size,
Reader: file,
})
if err != nil {
respondDomainError(w, err)
return
}
httputil.WriteJSON(w, http.StatusOK, dataset)
}
// downloadSLD streams a dataset's stored SLD style file. Returns 404 if none
// has been uploaded yet.
func (h *DatasetHandler) downloadSLD(w http.ResponseWriter, r *http.Request) {
id, ok := parseUUIDParam(w, r, "id")
if !ok {
return
}
dataset, obj, err := h.svc.DownloadSLD(r.Context(), id)
if err != nil {
respondDomainError(w, err)
return
}
defer obj.Close()
contentType := "application/xml"
if dataset.SldContentType != nil && *dataset.SldContentType != "" {
contentType = *dataset.SldContentType
}
filename := "style.sld"
if dataset.SldFilename != nil && *dataset.SldFilename != "" {
filename = *dataset.SldFilename
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`)
if _, err := io.Copy(w, obj); err != nil {
// Headers are already sent; nothing useful to return to the client.
return
}
}
func (h *DatasetHandler) delete(w http.ResponseWriter, r *http.Request) { func (h *DatasetHandler) delete(w http.ResponseWriter, r *http.Request) {
id, ok := parseUUIDParam(w, r, "id") id, ok := parseUUIDParam(w, r, "id")
if !ok { if !ok {

View File

@@ -0,0 +1,12 @@
-- +goose Up
-- Optional SLD (Styled Layer Descriptor) style file for a dataset. Applies to
-- every file_type; a dataset may have at most one (uploading again replaces
-- it).
ALTER TABLE datasets ADD COLUMN sld_storage_key TEXT;
ALTER TABLE datasets ADD COLUMN sld_filename VARCHAR(255);
ALTER TABLE datasets ADD COLUMN sld_content_type VARCHAR(255);
-- +goose Down
ALTER TABLE datasets DROP COLUMN sld_storage_key;
ALTER TABLE datasets DROP COLUMN sld_filename;
ALTER TABLE datasets DROP COLUMN sld_content_type;

View File

@@ -0,0 +1,15 @@
-- +goose Up
-- Optional period end date, so a year_column mapping can represent a date
-- range (date..date_to) instead of a single point in time. NULL preserves the
-- pre-existing point-in-time behavior; the unique index treats a NULL date_to
-- as equal to date itself so point observations keep deduplicating exactly as
-- before.
ALTER TABLE dataset_observations ADD COLUMN date_to DATE;
DROP INDEX uq_dataset_observations;
CREATE UNIQUE INDEX uq_dataset_observations ON dataset_observations (dataset_id, kato_code, date, COALESCE(date_to, date));
-- +goose Down
DROP INDEX uq_dataset_observations;
ALTER TABLE dataset_observations DROP COLUMN date_to;
CREATE UNIQUE INDEX uq_dataset_observations ON dataset_observations (dataset_id, kato_code, date);