feat: Add SLD endpoints, add date_to to vector_with_kato
This commit is contained in:
@@ -74,15 +74,19 @@ type DatasetFilter struct {
|
||||
}
|
||||
|
||||
// Observation is a single unpivoted value from a dataset's attribute table,
|
||||
// keyed by KATO code and date. Exactly one of Value / ValueText is typically
|
||||
// set (numeric vs non-numeric cell); both may be nil for an empty cell.
|
||||
// keyed by KATO code and date (optionally a date..DateTo period). Exactly one
|
||||
// of Value / ValueText is typically set (numeric vs non-numeric cell); both
|
||||
// may be nil for an empty cell.
|
||||
type Observation struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
DatasetID uuid.UUID `json:"dataset_id"`
|
||||
KatoCode string `json:"kato_code"`
|
||||
Date string `json:"date"`
|
||||
Value *float64 `json:"value"`
|
||||
ValueText *string `json:"value_text"`
|
||||
// DateTo is the period's end date. Nil for a point-in-time observation
|
||||
// (pre-existing behavior).
|
||||
DateTo *string `json:"date_to,omitempty"`
|
||||
Value *float64 `json:"value"`
|
||||
ValueText *string `json:"value_text"`
|
||||
}
|
||||
|
||||
// allowedExtensions lists the accepted lowercase file extensions (including the
|
||||
@@ -134,10 +138,30 @@ func ValidateFileContent(ext string, head []byte) error {
|
||||
if b, ok := firstMeaningfulByte(head); !ok || b != '{' {
|
||||
return fmt.Errorf("file is not valid GeoJSON")
|
||||
}
|
||||
case ".sld", ".xml":
|
||||
// SLD is XML; a lightweight sniff for the opening tag.
|
||||
if b, ok := firstMeaningfulByte(head); !ok || b != '<' {
|
||||
return fmt.Errorf("file is not valid XML/SLD")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AllowedSLDExtensions lists the accepted lowercase extensions (including the
|
||||
// dot) for a dataset's SLD (Styled Layer Descriptor) style file.
|
||||
var AllowedSLDExtensions = []string{".sld", ".xml"}
|
||||
|
||||
// ExtensionAllowedForSLD reports whether ext (lowercase, with dot) is a valid
|
||||
// SLD file extension.
|
||||
func ExtensionAllowedForSLD(ext string) bool {
|
||||
for _, e := range AllowedSLDExtensions {
|
||||
if e == ext {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// firstMeaningfulByte returns the first non-whitespace byte after an optional
|
||||
// UTF-8 BOM.
|
||||
func firstMeaningfulByte(head []byte) (byte, bool) {
|
||||
@@ -160,11 +184,15 @@ type AttributeColumn struct {
|
||||
Samples []string `json:"samples,omitempty"`
|
||||
}
|
||||
|
||||
// YearColumn maps an attribute column to the date it represents,
|
||||
// e.g. {"column": "F_2023", "date": "2023-01-01"}.
|
||||
// YearColumn maps an attribute column to the date (or date period) it
|
||||
// represents, e.g. {"column": "F_2023", "date": "2023-01-01"} for a single
|
||||
// point in time, or {"column": "F_2020_2023", "date": "2020-01-01", "date_to":
|
||||
// "2023-12-31"} for a period. DateTo is optional; when nil the mapping is a
|
||||
// single point in time, preserving the original (pre-period) behavior.
|
||||
type YearColumn struct {
|
||||
Column string `json:"column"`
|
||||
Date string `json:"date"`
|
||||
Column string `json:"column"`
|
||||
Date string `json:"date"`
|
||||
DateTo *string `json:"date_to,omitempty"`
|
||||
}
|
||||
|
||||
// DatasetSummary is the lightweight view of a dataset used in list responses.
|
||||
@@ -196,6 +224,11 @@ type Dataset struct {
|
||||
FileType FileType `json:"file_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ContentType string `json:"content_type"`
|
||||
// SldStorageKey points to the dataset's SLD (Styled Layer Descriptor) style
|
||||
// file, if one has been uploaded. Nullable; applies to every file type.
|
||||
SldStorageKey *string `json:"sld_storage_key"`
|
||||
SldFilename *string `json:"sld_filename"`
|
||||
SldContentType *string `json:"sld_content_type"`
|
||||
// Properties holds tabular data extracted from the file (e.g. a shapefile's
|
||||
// attribute table). Nullable; not exposed in dataset responses.
|
||||
Properties json.RawMessage `json:"-"`
|
||||
|
||||
@@ -31,7 +31,7 @@ const datasetColumns = `id, category_id, name, description, unit, filename, stor
|
||||
CASE WHEN file_type = 'raster' AND geometry IS NOT NULL
|
||||
THEN ARRAY[ST_XMin(geometry), ST_YMin(geometry), ST_XMax(geometry), ST_YMax(geometry)]
|
||||
ELSE NULL END AS bbox,
|
||||
geojson,
|
||||
geojson, sld_storage_key, sld_filename, sld_content_type,
|
||||
created_at, updated_at`
|
||||
|
||||
func scanDataset(row pgx.Row) (domain.Dataset, error) {
|
||||
@@ -41,7 +41,8 @@ func scanDataset(row pgx.Row) (domain.Dataset, error) {
|
||||
&d.Filename, &d.StorageKey, &d.CogStorageKey, &d.FileType, &d.SizeBytes, &d.ContentType,
|
||||
&d.Properties, &d.Meta, &d.Automated, &d.Status,
|
||||
&d.AttributeColumns, &d.KatoColumn, &d.YearColumns, &d.ParseError,
|
||||
&d.Geometry, &d.BBox, &d.GeoJSON, &d.CreatedAt, &d.UpdatedAt,
|
||||
&d.Geometry, &d.BBox, &d.GeoJSON, &d.SldStorageKey, &d.SldFilename, &d.SldContentType,
|
||||
&d.CreatedAt, &d.UpdatedAt,
|
||||
)
|
||||
return d, err
|
||||
}
|
||||
@@ -228,6 +229,20 @@ func (r *DatasetRepository) SetGeoJSON(ctx context.Context, id uuid.UUID, geojso
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetSLD stores the dataset's SLD (Styled Layer Descriptor) style file
|
||||
// metadata, replacing any previous one, and returns the updated row.
|
||||
func (r *DatasetRepository) SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error) {
|
||||
row := r.pool.QueryRow(ctx,
|
||||
`UPDATE datasets
|
||||
SET sld_storage_key = $2, sld_filename = $3, sld_content_type = $4, updated_at = now()
|
||||
WHERE id = $1
|
||||
RETURNING `+datasetColumns,
|
||||
id, key, filename, contentType,
|
||||
)
|
||||
out, err := scanDataset(row)
|
||||
return out, mapError(err)
|
||||
}
|
||||
|
||||
// ClearGeometryAndGeoJSON nulls out a dataset's stored geometry and persisted
|
||||
// GeoJSON. Used before re-enqueuing a dataset's processing job so a stale
|
||||
// (possibly incorrect) geometry/GeoJSON is never served while the job is
|
||||
@@ -265,11 +280,19 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid observation date %q: %w", o.Date, err)
|
||||
}
|
||||
rows[i] = []any{datasetID, o.KatoCode, d, o.Value, o.ValueText}
|
||||
var dateTo any
|
||||
if o.DateTo != nil {
|
||||
dt, err := time.Parse("2006-01-02", *o.DateTo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid observation date_to %q: %w", *o.DateTo, err)
|
||||
}
|
||||
dateTo = dt
|
||||
}
|
||||
rows[i] = []any{datasetID, o.KatoCode, d, dateTo, o.Value, o.ValueText}
|
||||
}
|
||||
_, err := tx.CopyFrom(ctx,
|
||||
pgx.Identifier{"dataset_observations"},
|
||||
[]string{"dataset_id", "kato_code", "date", "value", "value_text"},
|
||||
[]string{"dataset_id", "kato_code", "date", "date_to", "value", "value_text"},
|
||||
pgx.CopyFromRows(rows),
|
||||
)
|
||||
if err != nil {
|
||||
@@ -280,11 +303,11 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u
|
||||
return mapError(tx.Commit(ctx))
|
||||
}
|
||||
|
||||
const observationColumns = `id, dataset_id, kato_code, to_char(date, 'YYYY-MM-DD') AS date, value, value_text`
|
||||
const observationColumns = `id, dataset_id, kato_code, to_char(date, 'YYYY-MM-DD') AS date, to_char(date_to, 'YYYY-MM-DD') AS date_to, value, value_text`
|
||||
|
||||
func scanObservation(row pgx.Row) (domain.Observation, error) {
|
||||
var o domain.Observation
|
||||
err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.Date, &o.Value, &o.ValueText)
|
||||
err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.Date, &o.DateTo, &o.Value, &o.ValueText)
|
||||
return o, err
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,8 @@ var (
|
||||
// ErrCOGNotReady is returned when a raster's Cloud-Optimized GeoTIFF has not
|
||||
// been produced yet.
|
||||
ErrCOGNotReady = errors.New("cog is not ready")
|
||||
// ErrSLDNotFound is returned when a dataset has no SLD style file uploaded.
|
||||
ErrSLDNotFound = fmt.Errorf("%w: dataset has no sld", domain.ErrNotFound)
|
||||
)
|
||||
|
||||
// DatasetRepository is the persistence behaviour DatasetService needs.
|
||||
@@ -47,6 +49,7 @@ type DatasetRepository interface {
|
||||
SetGeoJSON(ctx context.Context, id uuid.UUID, geojson []byte) error
|
||||
ClearGeometryAndGeoJSON(ctx context.Context, id uuid.UUID) error
|
||||
SaveMapping(ctx context.Context, id uuid.UUID, katoColumn string, years []domain.YearColumn) (domain.Dataset, error)
|
||||
SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error)
|
||||
ReplaceObservations(ctx context.Context, datasetID uuid.UUID, obs []domain.Observation) error
|
||||
ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error)
|
||||
CountObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string) (int, error)
|
||||
@@ -235,6 +238,82 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat
|
||||
return dataset, nil
|
||||
}
|
||||
|
||||
// UploadSLDInput carries an SLD (Styled Layer Descriptor) style file to attach
|
||||
// to an existing dataset.
|
||||
type UploadSLDInput struct {
|
||||
Filename string
|
||||
ContentType string
|
||||
Size int64
|
||||
Reader io.Reader
|
||||
}
|
||||
|
||||
// UploadSLD validates and stores an SLD style file for a dataset, replacing
|
||||
// any previously stored one. Applies to every file_type.
|
||||
func (s *DatasetService) UploadSLD(ctx context.Context, id uuid.UUID, in UploadSLDInput) (domain.Dataset, error) {
|
||||
dataset, err := s.repo.GetByID(ctx, id)
|
||||
if err != nil {
|
||||
return domain.Dataset{}, err
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(in.Filename))
|
||||
if !domain.ExtensionAllowedForSLD(ext) {
|
||||
return domain.Dataset{}, fmt.Errorf("%w: sld file must have one of these extensions: %s",
|
||||
domain.ErrValidation, strings.Join(domain.AllowedSLDExtensions, ", "))
|
||||
}
|
||||
|
||||
head := make([]byte, 512)
|
||||
n, err := io.ReadFull(in.Reader, head)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return domain.Dataset{}, fmt.Errorf("read sld upload: %w", err)
|
||||
}
|
||||
head = head[:n]
|
||||
if err := domain.ValidateFileContent(ext, head); err != nil {
|
||||
return domain.Dataset{}, fmt.Errorf("%w: %s", domain.ErrValidation, err)
|
||||
}
|
||||
content := io.MultiReader(bytes.NewReader(head), in.Reader)
|
||||
|
||||
key := deriveSLDKey(dataset.StorageKey, in.Filename)
|
||||
if err := s.store.Put(ctx, key, content, in.Size, in.ContentType); err != nil {
|
||||
return domain.Dataset{}, err
|
||||
}
|
||||
|
||||
updated, err := s.repo.SetSLD(ctx, id, key, in.Filename, in.ContentType)
|
||||
if err != nil {
|
||||
// Compensate: the row was not updated, so the object would be orphaned.
|
||||
_ = s.store.Remove(ctx, key)
|
||||
return domain.Dataset{}, err
|
||||
}
|
||||
|
||||
if dataset.SldStorageKey != nil && *dataset.SldStorageKey != key {
|
||||
_ = s.store.Remove(ctx, *dataset.SldStorageKey) // best-effort: drop the replaced file
|
||||
}
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
// DownloadSLD returns the dataset metadata and a reader for its stored SLD
|
||||
// style file. Fails with ErrSLDNotFound when none has been uploaded. The
|
||||
// caller must close the reader.
|
||||
func (s *DatasetService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain.Dataset, io.ReadCloser, error) {
|
||||
dataset, err := s.repo.GetByID(ctx, id)
|
||||
if err != nil {
|
||||
return domain.Dataset{}, nil, err
|
||||
}
|
||||
if dataset.SldStorageKey == nil {
|
||||
return domain.Dataset{}, nil, ErrSLDNotFound
|
||||
}
|
||||
obj, err := s.store.Get(ctx, *dataset.SldStorageKey)
|
||||
if err != nil {
|
||||
return domain.Dataset{}, nil, err
|
||||
}
|
||||
return dataset, obj, nil
|
||||
}
|
||||
|
||||
// deriveSLDKey places a dataset's SLD style file alongside the original under
|
||||
// an sld/ prefix, named after the uploaded SLD file.
|
||||
func deriveSLDKey(storageKey, filename string) string {
|
||||
return path.Join(path.Dir(storageKey), "sld", filename)
|
||||
}
|
||||
|
||||
// enqueueProcessing schedules the appropriate async job for a dataset based on
|
||||
// its file type: vector_with_kato is parsed for column selection, plain vector
|
||||
// has its attribute table extracted into properties, and raster is converted to
|
||||
@@ -603,9 +682,19 @@ func (s *DatasetService) SaveMapping(ctx context.Context, id uuid.UUID, in Mappi
|
||||
if _, ok := known[yc.Column]; !ok {
|
||||
return domain.Dataset{}, fmt.Errorf("%w: year column %q is not among the detected columns", domain.ErrValidation, yc.Column)
|
||||
}
|
||||
if _, err := time.Parse("2006-01-02", yc.Date); err != nil {
|
||||
from, err := time.Parse("2006-01-02", yc.Date)
|
||||
if err != nil {
|
||||
return domain.Dataset{}, fmt.Errorf("%w: invalid date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.Date, yc.Column)
|
||||
}
|
||||
if yc.DateTo != nil {
|
||||
to, err := time.Parse("2006-01-02", *yc.DateTo)
|
||||
if err != nil {
|
||||
return domain.Dataset{}, fmt.Errorf("%w: invalid date_to %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, *yc.DateTo, yc.Column)
|
||||
}
|
||||
if to.Before(from) {
|
||||
return domain.Dataset{}, fmt.Errorf("%w: date_to %q is before date %q for column %q", domain.ErrValidation, *yc.DateTo, yc.Date, yc.Column)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dataset, err = s.repo.SaveMapping(ctx, id, in.KatoColumn, in.YearColumns)
|
||||
@@ -659,8 +748,9 @@ func (s *DatasetService) Extract(ctx context.Context, id uuid.UUID) error {
|
||||
}
|
||||
|
||||
// buildObservations unpivots rows into observations. Rows without a KATO code
|
||||
// are skipped; duplicate (kato, date) pairs keep the last value. Numeric cells
|
||||
// populate Value, others ValueText.
|
||||
// are skipped; duplicate (kato, date, date_to) tuples keep the last value.
|
||||
// Numeric cells populate Value, others ValueText. A year column's DateTo (nil
|
||||
// for a point-in-time mapping) carries through to the observation unchanged.
|
||||
func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.YearColumn, rows []map[string]string) []domain.Observation {
|
||||
obs := make([]domain.Observation, 0, len(rows)*len(years))
|
||||
index := make(map[string]int)
|
||||
@@ -671,7 +761,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
|
||||
continue
|
||||
}
|
||||
for _, yc := range years {
|
||||
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date}
|
||||
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date, DateTo: yc.DateTo}
|
||||
if raw := strings.TrimSpace(row[yc.Column]); raw != "" {
|
||||
if f, err := strconv.ParseFloat(raw, 64); err == nil {
|
||||
o.Value = &f
|
||||
@@ -679,7 +769,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
|
||||
o.ValueText = &raw
|
||||
}
|
||||
}
|
||||
key := kato + "\x00" + yc.Date
|
||||
key := kato + "\x00" + yc.Date + "\x00" + dateToKey(yc.DateTo)
|
||||
if i, ok := index[key]; ok {
|
||||
obs[i] = o
|
||||
} else {
|
||||
@@ -691,6 +781,14 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
|
||||
return obs
|
||||
}
|
||||
|
||||
// dateToKey normalizes an optional period end date for use in a dedup key.
|
||||
func dateToKey(dateTo *string) string {
|
||||
if dateTo == nil {
|
||||
return ""
|
||||
}
|
||||
return *dateTo
|
||||
}
|
||||
|
||||
// ObservationPage is a page of observations with pagination metadata.
|
||||
type ObservationPage struct {
|
||||
Items []domain.Observation
|
||||
@@ -789,7 +887,11 @@ func (s *DatasetService) generateGeoJSON(ctx context.Context, id uuid.UUID) erro
|
||||
// joins the districts table on KATO code: one Feature is emitted per KATO, its
|
||||
// boundary taken from the districts table and its observation values nested
|
||||
// under a `data` object (keyed by date) alongside `kato` and `name`. KATO codes
|
||||
// with no matching district are skipped.
|
||||
// with no matching district are skipped. When any observation carries a period
|
||||
// (DateTo set), its end date is additionally exposed under a `periods` object
|
||||
// (keyed by the same start date used in `data`); `periods` is omitted entirely
|
||||
// when every observation is point-in-time, so a dataset mapped without periods
|
||||
// produces byte-identical output to before periods existed.
|
||||
func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.RawMessage, error) {
|
||||
dataset, err := s.repo.GetByID(ctx, id)
|
||||
if err != nil {
|
||||
@@ -816,8 +918,8 @@ func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.R
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
grouped, order := groupObservationsByKato(obs)
|
||||
features, err := s.districtFeatures(ctx, grouped, order)
|
||||
grouped, periods, order := groupObservationsByKato(obs)
|
||||
features, err := s.districtFeatures(ctx, grouped, periods, order)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -850,9 +952,12 @@ func (s *DatasetService) loadGeoJSONDataset(ctx context.Context, id uuid.UUID) (
|
||||
|
||||
// districtFeatures builds one Feature per KATO from the districts table, with
|
||||
// the grouped observation values (keyed by date) nested under a `data` object
|
||||
// alongside `kato` and `name` in each Feature's properties. KATO codes with no
|
||||
// alongside `kato` and `name` in each Feature's properties. When periods holds
|
||||
// an entry for the KATO (at least one observation had a period), its date->
|
||||
// date_to map is additionally nested under a `periods` object; a KATO with no
|
||||
// period observations gets no `periods` key at all. KATO codes with no
|
||||
// matching district are skipped. order drives the deterministic feature order.
|
||||
func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, order []string) ([]domain.Feature, error) {
|
||||
func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, periods map[string]map[string]string, order []string) ([]domain.Feature, error) {
|
||||
districts, err := s.repo.DistrictGeometriesByKato(ctx, order)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -863,10 +968,14 @@ func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[strin
|
||||
if !ok {
|
||||
continue // skip KATO codes with no district boundary
|
||||
}
|
||||
props := map[string]any{"kato": kato, "name": dist.Name, "data": grouped[kato]}
|
||||
if p, ok := periods[kato]; ok {
|
||||
props["periods"] = p
|
||||
}
|
||||
features = append(features, domain.Feature{
|
||||
Type: domain.GeoJSONFeature,
|
||||
Geometry: dist.Geometry,
|
||||
Properties: map[string]any{"kato": kato, "name": dist.Name, "data": grouped[kato]},
|
||||
Properties: props,
|
||||
})
|
||||
}
|
||||
return features, nil
|
||||
@@ -909,11 +1018,15 @@ func vectorFeatureProperties(raw json.RawMessage) map[string]any {
|
||||
|
||||
// groupObservationsByKato groups observations by KATO code into date->value
|
||||
// maps. The value is the numeric Value when present, otherwise ValueText,
|
||||
// otherwise nil (an empty cell). order lists KATO codes in first-seen order so
|
||||
// the resulting feature order is deterministic.
|
||||
func groupObservationsByKato(obs []domain.Observation) (map[string]map[string]any, []string) {
|
||||
grouped := make(map[string]map[string]any)
|
||||
order := make([]string, 0)
|
||||
// otherwise nil (an empty cell). It also groups, per KATO, a date->date_to map
|
||||
// for observations that carry a period (DateTo set); a KATO with no period
|
||||
// observations gets no entry in periods at all, so callers can tell "no
|
||||
// periods" apart from "empty periods". order lists KATO codes in first-seen
|
||||
// order so the resulting feature order is deterministic.
|
||||
func groupObservationsByKato(obs []domain.Observation) (grouped map[string]map[string]any, periods map[string]map[string]string, order []string) {
|
||||
grouped = make(map[string]map[string]any)
|
||||
periods = make(map[string]map[string]string)
|
||||
order = make([]string, 0)
|
||||
for _, o := range obs {
|
||||
values, ok := grouped[o.KatoCode]
|
||||
if !ok {
|
||||
@@ -922,8 +1035,17 @@ func groupObservationsByKato(obs []domain.Observation) (map[string]map[string]an
|
||||
order = append(order, o.KatoCode)
|
||||
}
|
||||
values[o.Date] = observationValue(o)
|
||||
|
||||
if o.DateTo != nil {
|
||||
p, ok := periods[o.KatoCode]
|
||||
if !ok {
|
||||
p = make(map[string]string)
|
||||
periods[o.KatoCode] = p
|
||||
}
|
||||
p[o.Date] = *o.DateTo
|
||||
}
|
||||
}
|
||||
return grouped, order
|
||||
return grouped, periods, order
|
||||
}
|
||||
|
||||
// observationValue returns the typed cell value: numeric Value, else ValueText,
|
||||
|
||||
@@ -137,6 +137,18 @@ func (r *stubDatasetRepo) SaveMapping(_ context.Context, id uuid.UUID, kato stri
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (r *stubDatasetRepo) SetSLD(_ context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error) {
|
||||
d, ok := r.store[id]
|
||||
if !ok {
|
||||
return domain.Dataset{}, domain.ErrNotFound
|
||||
}
|
||||
d.SldStorageKey = &key
|
||||
d.SldFilename = &filename
|
||||
d.SldContentType = &contentType
|
||||
r.store[id] = d
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (r *stubDatasetRepo) MarkReady(_ context.Context, id uuid.UUID, geometry []byte) error {
|
||||
d, ok := r.store[id]
|
||||
if !ok {
|
||||
@@ -776,6 +788,39 @@ func TestDatasetService_SaveMapping(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("bad date_to", func(t *testing.T) {
|
||||
_, svc := newSvc()
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-01-01", DateTo: strPtr("2023")}}})
|
||||
if !errors.Is(err, domain.ErrValidation) {
|
||||
t.Fatalf("want ErrValidation, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("date_to before date", func(t *testing.T) {
|
||||
_, svc := newSvc()
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-12-31", DateTo: strPtr("2023-01-01")}}})
|
||||
if !errors.Is(err, domain.ErrValidation) {
|
||||
t.Fatalf("want ErrValidation, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("period mapping succeeds", func(t *testing.T) {
|
||||
repo, svc := newSvc()
|
||||
got, err := svc.SaveMapping(ctx, id, MappingInput{
|
||||
KatoColumn: "като",
|
||||
YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2020-01-01", DateTo: strPtr("2023-12-31")}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(got.YearColumns) != 1 || got.YearColumns[0].DateTo == nil || *got.YearColumns[0].DateTo != "2023-12-31" {
|
||||
t.Fatalf("period not saved: %+v", got.YearColumns)
|
||||
}
|
||||
if len(repo.store[id].YearColumns) != 1 {
|
||||
t.Fatalf("period not persisted: %+v", repo.store[id].YearColumns)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no year columns", func(t *testing.T) {
|
||||
_, svc := newSvc()
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като"})
|
||||
@@ -845,6 +890,45 @@ func TestBuildObservations(t *testing.T) {
|
||||
if o := byKey["751010000|2025-01-01"]; o.ValueText == nil || *o.ValueText != "n/a" {
|
||||
t.Errorf("non-numeric cell not stored as value_text: %+v", o)
|
||||
}
|
||||
for _, o := range obs {
|
||||
if o.DateTo != nil {
|
||||
t.Fatalf("point-in-time mapping must not set DateTo, got %+v", o)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildObservations_Period(t *testing.T) {
|
||||
id := uuid.New()
|
||||
years := []domain.YearColumn{
|
||||
{Column: "F_2020_2023", Date: "2020-01-01", DateTo: strPtr("2023-12-31")}, // period
|
||||
{Column: "F_2024", Date: "2024-01-01"}, // point-in-time, unaffected
|
||||
}
|
||||
rows := []map[string]string{
|
||||
{"като": "751010000", "F_2020_2023": "100", "F_2024": "200"},
|
||||
}
|
||||
|
||||
obs := buildObservations(id, "като", years, rows)
|
||||
if len(obs) != 2 {
|
||||
t.Fatalf("want 2 observations, got %d", len(obs))
|
||||
}
|
||||
|
||||
byDate := map[string]domain.Observation{}
|
||||
for _, o := range obs {
|
||||
byDate[o.Date] = o
|
||||
}
|
||||
|
||||
period := byDate["2020-01-01"]
|
||||
if period.DateTo == nil || *period.DateTo != "2023-12-31" {
|
||||
t.Fatalf("period observation missing date_to: %+v", period)
|
||||
}
|
||||
if period.Value == nil || *period.Value != 100 {
|
||||
t.Fatalf("period observation value wrong: %+v", period)
|
||||
}
|
||||
|
||||
point := byDate["2024-01-01"]
|
||||
if point.DateTo != nil {
|
||||
t.Fatalf("point-in-time observation must have nil date_to (backward compat), got %+v", point)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_Extract(t *testing.T) {
|
||||
@@ -959,6 +1043,68 @@ func TestDatasetService_GeoJSON_DistrictJoin(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_GeoJSON_Period(t *testing.T) {
|
||||
id := uuid.New()
|
||||
repo := newStubDatasetRepo()
|
||||
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeVectorWithKato, Status: domain.DatasetStatusReady}
|
||||
|
||||
period, point := 100.0, 200.0
|
||||
repo.observations[id] = []domain.Observation{
|
||||
{KatoCode: "710000000", Date: "2020-01-01", DateTo: strPtr("2023-12-31"), Value: &period},
|
||||
{KatoCode: "710000000", Date: "2024-01-01", Value: &point}, // point-in-time, no period
|
||||
}
|
||||
repo.districts["710000000"] = domain.District{
|
||||
Kato: "710000000", Name: "Astana",
|
||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`),
|
||||
}
|
||||
|
||||
svc := newDatasetService(repo, &stubStore{}, true)
|
||||
fc := mustGeoJSON(t, svc, id)
|
||||
if len(fc.Features) != 1 {
|
||||
t.Fatalf("want 1 feature, got %d", len(fc.Features))
|
||||
}
|
||||
f := fc.Features[0]
|
||||
|
||||
data, ok := f.Properties["data"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("observations not nested under data: %+v", f.Properties)
|
||||
}
|
||||
if data["2020-01-01"] != 100.0 || data["2024-01-01"] != 200.0 {
|
||||
t.Fatalf("unexpected data: %+v", data)
|
||||
}
|
||||
|
||||
periods, ok := f.Properties["periods"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("periods not exposed as a map: %+v", f.Properties)
|
||||
}
|
||||
if len(periods) != 1 || periods["2020-01-01"] != "2023-12-31" {
|
||||
t.Fatalf("period end date not exposed: %+v", periods)
|
||||
}
|
||||
if _, has := periods["2024-01-01"]; has {
|
||||
t.Fatalf("point-in-time date must not appear in periods: %+v", periods)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_GeoJSON_NoPeriods_OmitsPeriodsKey(t *testing.T) {
|
||||
// Regression: a dataset mapped without any period must produce output
|
||||
// byte-identical to before periods existed - no "periods" key at all.
|
||||
id := uuid.New()
|
||||
repo := newStubDatasetRepo()
|
||||
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeVectorWithKato, Status: domain.DatasetStatusReady}
|
||||
v := 100.0
|
||||
repo.observations[id] = []domain.Observation{{KatoCode: "710000000", Date: "2020-01-01", Value: &v}}
|
||||
repo.districts["710000000"] = domain.District{
|
||||
Kato: "710000000", Name: "Astana",
|
||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`),
|
||||
}
|
||||
|
||||
svc := newDatasetService(repo, &stubStore{}, true)
|
||||
fc := mustGeoJSON(t, svc, id)
|
||||
if _, has := fc.Features[0].Properties["periods"]; has {
|
||||
t.Fatalf("periods key must be absent when no observation has a period: %+v", fc.Features[0].Properties)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_GeoJSON_IgnoresDatasetGeometry(t *testing.T) {
|
||||
id := uuid.New()
|
||||
repo := newStubDatasetRepo()
|
||||
@@ -1294,6 +1440,125 @@ func TestDatasetService_ReprocessAll_FiltersByFileType(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func validSLDUpload() UploadSLDInput {
|
||||
body := `<?xml version="1.0"?><StyledLayerDescriptor></StyledLayerDescriptor>`
|
||||
return UploadSLDInput{
|
||||
Filename: "style.sld",
|
||||
ContentType: "application/xml",
|
||||
Size: int64(len(body)),
|
||||
Reader: strings.NewReader(body),
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_UploadSLD_Success(t *testing.T) {
|
||||
id := uuid.New()
|
||||
repo := newStubDatasetRepo()
|
||||
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeRaster, StorageKey: "uid/dem.tif"}
|
||||
store := &stubStore{}
|
||||
svc := newDatasetService(repo, store, true)
|
||||
|
||||
got, err := svc.UploadSLD(context.Background(), id, validSLDUpload())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got.SldStorageKey == nil || *got.SldStorageKey != "uid/sld/style.sld" {
|
||||
t.Fatalf("unexpected sld storage key: %v", got.SldStorageKey)
|
||||
}
|
||||
if got.SldFilename == nil || *got.SldFilename != "style.sld" {
|
||||
t.Fatalf("sld filename not stored: %v", got.SldFilename)
|
||||
}
|
||||
if len(store.put) != 1 || store.put[0] != "uid/sld/style.sld" {
|
||||
t.Fatalf("sld not uploaded to store: %v", store.put)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_UploadSLD_ReplacesPrevious(t *testing.T) {
|
||||
id := uuid.New()
|
||||
repo := newStubDatasetRepo()
|
||||
oldKey := "uid/sld/old.sld"
|
||||
repo.store[id] = domain.Dataset{
|
||||
ID: id, FileType: domain.FileTypeVector, StorageKey: "uid/data.geojson",
|
||||
SldStorageKey: &oldKey,
|
||||
}
|
||||
store := &stubStore{}
|
||||
svc := newDatasetService(repo, store, true)
|
||||
|
||||
in := validSLDUpload()
|
||||
in.Filename = "new.sld"
|
||||
got, err := svc.UploadSLD(context.Background(), id, in)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got.SldStorageKey == nil || *got.SldStorageKey != "uid/sld/new.sld" {
|
||||
t.Fatalf("unexpected sld storage key: %v", got.SldStorageKey)
|
||||
}
|
||||
if len(store.removed) != 1 || store.removed[0] != oldKey {
|
||||
t.Fatalf("old sld object not removed: %v", store.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_UploadSLD_RejectsBadExtension(t *testing.T) {
|
||||
id := uuid.New()
|
||||
repo := newStubDatasetRepo()
|
||||
repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"}
|
||||
store := &stubStore{}
|
||||
svc := newDatasetService(repo, store, true)
|
||||
|
||||
in := validSLDUpload()
|
||||
in.Filename = "style.zip"
|
||||
_, err := svc.UploadSLD(context.Background(), id, in)
|
||||
if !errors.Is(err, domain.ErrValidation) {
|
||||
t.Fatalf("want ErrValidation, got %v", err)
|
||||
}
|
||||
if len(store.put) != 0 {
|
||||
t.Fatalf("nothing should be uploaded on validation failure, got %v", store.put)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_UploadSLD_RejectsMismatchedContent(t *testing.T) {
|
||||
id := uuid.New()
|
||||
repo := newStubDatasetRepo()
|
||||
repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"}
|
||||
store := &stubStore{}
|
||||
svc := newDatasetService(repo, store, true)
|
||||
|
||||
in := validSLDUpload()
|
||||
in.Reader = strings.NewReader(`{"not":"xml"}`)
|
||||
_, err := svc.UploadSLD(context.Background(), id, in)
|
||||
if !errors.Is(err, domain.ErrValidation) {
|
||||
t.Fatalf("want ErrValidation, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_DownloadSLD(t *testing.T) {
|
||||
id := uuid.New()
|
||||
repo := newStubDatasetRepo()
|
||||
repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"}
|
||||
store := &stubStore{}
|
||||
svc := newDatasetService(repo, store, true)
|
||||
|
||||
t.Run("not found before upload", func(t *testing.T) {
|
||||
_, _, err := svc.DownloadSLD(context.Background(), id)
|
||||
if !errors.Is(err, ErrSLDNotFound) {
|
||||
t.Fatalf("want ErrSLDNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("streams after upload", func(t *testing.T) {
|
||||
if _, err := svc.UploadSLD(context.Background(), id, validSLDUpload()); err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
_, obj, err := svc.DownloadSLD(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
defer obj.Close()
|
||||
if _, err := io.ReadAll(obj); err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDatasetService_Delete_RemovesObject(t *testing.T) {
|
||||
repo := newStubDatasetRepo()
|
||||
id := uuid.New()
|
||||
|
||||
@@ -43,6 +43,8 @@ func (h *DatasetHandler) Register(r chi.Router) {
|
||||
r.Get("/{id}.cog", h.cog)
|
||||
r.Get("/{id}/status", h.status)
|
||||
r.Get("/{id}/download", h.download)
|
||||
r.Post("/{id}/sld", h.uploadSLD)
|
||||
r.Get("/{id}/sld", h.downloadSLD)
|
||||
r.Post("/{id}/mapping", h.mapping)
|
||||
r.Get("/{id}/observations", h.observations)
|
||||
r.Delete("/{id}", h.delete)
|
||||
@@ -78,8 +80,9 @@ func (h *DatasetHandler) status(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
type yearColumnInput struct {
|
||||
Column string `json:"column" validate:"required"`
|
||||
Date string `json:"date" validate:"required,datetime=2006-01-02"`
|
||||
Column string `json:"column" validate:"required"`
|
||||
Date string `json:"date" validate:"required,datetime=2006-01-02"`
|
||||
DateTo *string `json:"date_to" validate:"omitempty,datetime=2006-01-02"`
|
||||
}
|
||||
|
||||
type mappingRequest struct {
|
||||
@@ -106,7 +109,7 @@ func (h *DatasetHandler) mapping(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
in := service.MappingInput{KatoColumn: req.KatoColumn}
|
||||
for _, yc := range req.YearColumns {
|
||||
in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, Date: yc.Date})
|
||||
in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, Date: yc.Date, DateTo: yc.DateTo})
|
||||
}
|
||||
|
||||
dataset, err := h.svc.SaveMapping(r.Context(), id, in)
|
||||
@@ -533,6 +536,74 @@ func (h *DatasetHandler) download(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// uploadSLD stores (or replaces) a dataset's SLD (Styled Layer Descriptor)
|
||||
// style file. Multipart upload, mirroring POST /datasets. Applies to every
|
||||
// file_type.
|
||||
func (h *DatasetHandler) uploadSLD(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := parseUUIDParam(w, r, "id")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
|
||||
httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data")
|
||||
return
|
||||
}
|
||||
|
||||
file, header, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
httputil.WriteError(w, http.StatusBadRequest, "file is required")
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
contentType := header.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "application/xml"
|
||||
}
|
||||
|
||||
dataset, err := h.svc.UploadSLD(r.Context(), id, service.UploadSLDInput{
|
||||
Filename: header.Filename,
|
||||
ContentType: contentType,
|
||||
Size: header.Size,
|
||||
Reader: file,
|
||||
})
|
||||
if err != nil {
|
||||
respondDomainError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, dataset)
|
||||
}
|
||||
|
||||
// downloadSLD streams a dataset's stored SLD style file. Returns 404 if none
|
||||
// has been uploaded yet.
|
||||
func (h *DatasetHandler) downloadSLD(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := parseUUIDParam(w, r, "id")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
dataset, obj, err := h.svc.DownloadSLD(r.Context(), id)
|
||||
if err != nil {
|
||||
respondDomainError(w, err)
|
||||
return
|
||||
}
|
||||
defer obj.Close()
|
||||
|
||||
contentType := "application/xml"
|
||||
if dataset.SldContentType != nil && *dataset.SldContentType != "" {
|
||||
contentType = *dataset.SldContentType
|
||||
}
|
||||
filename := "style.sld"
|
||||
if dataset.SldFilename != nil && *dataset.SldFilename != "" {
|
||||
filename = *dataset.SldFilename
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`)
|
||||
if _, err := io.Copy(w, obj); err != nil {
|
||||
// Headers are already sent; nothing useful to return to the client.
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *DatasetHandler) delete(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := parseUUIDParam(w, r, "id")
|
||||
if !ok {
|
||||
|
||||
Reference in New Issue
Block a user