From 1e0844ca82950238f7334c594daa938700cfda73 Mon Sep 17 00:00:00 2001 From: Bakhtiyar Issakhmetov Date: Sat, 1 Aug 2026 22:09:59 +0500 Subject: [PATCH] feat: Add SLD endpoints, add date_to to vector_with_kato --- api/openapi.yaml | 81 ++++++ internal/domain/dataset.go | 49 +++- internal/repository/postgres/dataset.go | 35 ++- internal/service/dataset.go | 156 +++++++++-- internal/service/dataset_test.go | 265 ++++++++++++++++++ internal/transport/http/dataset_handler.go | 77 ++++- migrations/00008_add_sld_to_datasets.sql | 12 + ...009_add_period_to_dataset_observations.sql | 15 + 8 files changed, 656 insertions(+), 34 deletions(-) create mode 100644 migrations/00008_add_sld_to_datasets.sql create mode 100644 migrations/00009_add_period_to_dataset_observations.sql diff --git a/api/openapi.yaml b/api/openapi.yaml index 1ea78b6..103bf2a 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -432,6 +432,54 @@ paths: "400": { $ref: "#/components/responses/BadRequest" } "404": { $ref: "#/components/responses/NotFound" } + /datasets/{id}/sld: + parameters: + - $ref: "#/components/parameters/IdParam" + post: + tags: [Datasets] + summary: Upload (or replace) a dataset's SLD style file + description: | + Multipart upload of an SLD (Styled Layer Descriptor) style file, + mirroring `POST /datasets`. Applies to every `file_type`. Uploading + again replaces the previously stored SLD. + + Allowed extensions: `.sld`, `.xml`. + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: [file] + properties: + file: + type: string + format: binary + description: The SLD file. + responses: + "200": + description: SLD stored; the updated dataset + content: + application/json: + schema: { $ref: "#/components/schemas/Dataset" } + "400": { $ref: "#/components/responses/BadRequest" } + "404": { $ref: "#/components/responses/NotFound" } + "422": { $ref: "#/components/responses/ValidationError" } + get: + tags: [Datasets] + summary: Download a dataset's SLD style file + description: Returns 404 if no SLD has been uploaded for this dataset yet. + responses: + "200": + description: The SLD file stream + content: + application/xml: + schema: + type: string + format: binary + "400": { $ref: "#/components/responses/BadRequest" } + "404": { $ref: "#/components/responses/NotFound" } + /datasets/{id}/mapping: parameters: - $ref: "#/components/parameters/IdParam" @@ -443,6 +491,10 @@ paths: `ready`, to re-map). The KATO column and every mapped column must be among the dataset's detected `attribute_columns`. Moves the dataset to `extracting`; observations are produced asynchronously. + + Each year column maps to either a single point in time (`date` only, + the original behavior) or a period (`date` and `date_to`, inclusive). + `date_to`, when present, must not be before `date`. requestBody: required: true content: @@ -672,6 +724,16 @@ components: date: type: string format: date + description: >- + Point in time, or the start of the period when `date_to` is also + set. + date_to: + type: [string, "null"] + format: date + description: >- + Optional period end (inclusive), must not be before `date`. Omitted + or null means a single point in time (the original, pre-period + behavior). GeoJSONGeometry: type: [object, "null"] @@ -699,6 +761,12 @@ components: date (YYYY-MM-DD). For the single-feature case (dataset has its own geometry) it holds only the observations, keyed by KATO code, each mapping to its date->value pairs. + + For a `vector_with_kato` dataset mapped with one or more period + year columns (`date_to` set), a sibling `periods` object is also + present, mapping each period's start date (the same key used in + `data`) to its end date. `periods` is omitted entirely when the + dataset has no period mappings. additionalProperties: true GeoJSONFeatureCollection: @@ -769,6 +837,13 @@ components: cog_storage_key: type: [string, "null"] description: Set for rasters once the Cloud-Optimized GeoTIFF is produced. + sld_storage_key: + type: [string, "null"] + description: Set once an SLD style file has been uploaded (any file_type). + sld_filename: + type: [string, "null"] + sld_content_type: + type: [string, "null"] file_type: $ref: "#/components/schemas/FileType" size_bytes: @@ -844,6 +919,12 @@ components: date: type: string format: date + date_to: + type: [string, "null"] + format: date + description: >- + Period end (inclusive), carried over from the year mapping's + `date_to`. Null for a point-in-time observation. value: type: [number, "null"] description: Numeric cell value, or null when non-numeric/empty. diff --git a/internal/domain/dataset.go b/internal/domain/dataset.go index d461914..fb01378 100644 --- a/internal/domain/dataset.go +++ b/internal/domain/dataset.go @@ -74,15 +74,19 @@ type DatasetFilter struct { } // Observation is a single unpivoted value from a dataset's attribute table, -// keyed by KATO code and date. Exactly one of Value / ValueText is typically -// set (numeric vs non-numeric cell); both may be nil for an empty cell. +// keyed by KATO code and date (optionally a date..DateTo period). Exactly one +// of Value / ValueText is typically set (numeric vs non-numeric cell); both +// may be nil for an empty cell. type Observation struct { ID uuid.UUID `json:"id"` DatasetID uuid.UUID `json:"dataset_id"` KatoCode string `json:"kato_code"` Date string `json:"date"` - Value *float64 `json:"value"` - ValueText *string `json:"value_text"` + // DateTo is the period's end date. Nil for a point-in-time observation + // (pre-existing behavior). + DateTo *string `json:"date_to,omitempty"` + Value *float64 `json:"value"` + ValueText *string `json:"value_text"` } // allowedExtensions lists the accepted lowercase file extensions (including the @@ -134,10 +138,30 @@ func ValidateFileContent(ext string, head []byte) error { if b, ok := firstMeaningfulByte(head); !ok || b != '{' { return fmt.Errorf("file is not valid GeoJSON") } + case ".sld", ".xml": + // SLD is XML; a lightweight sniff for the opening tag. + if b, ok := firstMeaningfulByte(head); !ok || b != '<' { + return fmt.Errorf("file is not valid XML/SLD") + } } return nil } +// AllowedSLDExtensions lists the accepted lowercase extensions (including the +// dot) for a dataset's SLD (Styled Layer Descriptor) style file. +var AllowedSLDExtensions = []string{".sld", ".xml"} + +// ExtensionAllowedForSLD reports whether ext (lowercase, with dot) is a valid +// SLD file extension. +func ExtensionAllowedForSLD(ext string) bool { + for _, e := range AllowedSLDExtensions { + if e == ext { + return true + } + } + return false +} + // firstMeaningfulByte returns the first non-whitespace byte after an optional // UTF-8 BOM. func firstMeaningfulByte(head []byte) (byte, bool) { @@ -160,11 +184,15 @@ type AttributeColumn struct { Samples []string `json:"samples,omitempty"` } -// YearColumn maps an attribute column to the date it represents, -// e.g. {"column": "F_2023", "date": "2023-01-01"}. +// YearColumn maps an attribute column to the date (or date period) it +// represents, e.g. {"column": "F_2023", "date": "2023-01-01"} for a single +// point in time, or {"column": "F_2020_2023", "date": "2020-01-01", "date_to": +// "2023-12-31"} for a period. DateTo is optional; when nil the mapping is a +// single point in time, preserving the original (pre-period) behavior. type YearColumn struct { - Column string `json:"column"` - Date string `json:"date"` + Column string `json:"column"` + Date string `json:"date"` + DateTo *string `json:"date_to,omitempty"` } // DatasetSummary is the lightweight view of a dataset used in list responses. @@ -196,6 +224,11 @@ type Dataset struct { FileType FileType `json:"file_type"` SizeBytes int64 `json:"size_bytes"` ContentType string `json:"content_type"` + // SldStorageKey points to the dataset's SLD (Styled Layer Descriptor) style + // file, if one has been uploaded. Nullable; applies to every file type. + SldStorageKey *string `json:"sld_storage_key"` + SldFilename *string `json:"sld_filename"` + SldContentType *string `json:"sld_content_type"` // Properties holds tabular data extracted from the file (e.g. a shapefile's // attribute table). Nullable; not exposed in dataset responses. Properties json.RawMessage `json:"-"` diff --git a/internal/repository/postgres/dataset.go b/internal/repository/postgres/dataset.go index 08695fe..79db5ae 100644 --- a/internal/repository/postgres/dataset.go +++ b/internal/repository/postgres/dataset.go @@ -31,7 +31,7 @@ const datasetColumns = `id, category_id, name, description, unit, filename, stor CASE WHEN file_type = 'raster' AND geometry IS NOT NULL THEN ARRAY[ST_XMin(geometry), ST_YMin(geometry), ST_XMax(geometry), ST_YMax(geometry)] ELSE NULL END AS bbox, - geojson, + geojson, sld_storage_key, sld_filename, sld_content_type, created_at, updated_at` func scanDataset(row pgx.Row) (domain.Dataset, error) { @@ -41,7 +41,8 @@ func scanDataset(row pgx.Row) (domain.Dataset, error) { &d.Filename, &d.StorageKey, &d.CogStorageKey, &d.FileType, &d.SizeBytes, &d.ContentType, &d.Properties, &d.Meta, &d.Automated, &d.Status, &d.AttributeColumns, &d.KatoColumn, &d.YearColumns, &d.ParseError, - &d.Geometry, &d.BBox, &d.GeoJSON, &d.CreatedAt, &d.UpdatedAt, + &d.Geometry, &d.BBox, &d.GeoJSON, &d.SldStorageKey, &d.SldFilename, &d.SldContentType, + &d.CreatedAt, &d.UpdatedAt, ) return d, err } @@ -228,6 +229,20 @@ func (r *DatasetRepository) SetGeoJSON(ctx context.Context, id uuid.UUID, geojso return nil } +// SetSLD stores the dataset's SLD (Styled Layer Descriptor) style file +// metadata, replacing any previous one, and returns the updated row. +func (r *DatasetRepository) SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error) { + row := r.pool.QueryRow(ctx, + `UPDATE datasets + SET sld_storage_key = $2, sld_filename = $3, sld_content_type = $4, updated_at = now() + WHERE id = $1 + RETURNING `+datasetColumns, + id, key, filename, contentType, + ) + out, err := scanDataset(row) + return out, mapError(err) +} + // ClearGeometryAndGeoJSON nulls out a dataset's stored geometry and persisted // GeoJSON. Used before re-enqueuing a dataset's processing job so a stale // (possibly incorrect) geometry/GeoJSON is never served while the job is @@ -265,11 +280,19 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u if err != nil { return fmt.Errorf("invalid observation date %q: %w", o.Date, err) } - rows[i] = []any{datasetID, o.KatoCode, d, o.Value, o.ValueText} + var dateTo any + if o.DateTo != nil { + dt, err := time.Parse("2006-01-02", *o.DateTo) + if err != nil { + return fmt.Errorf("invalid observation date_to %q: %w", *o.DateTo, err) + } + dateTo = dt + } + rows[i] = []any{datasetID, o.KatoCode, d, dateTo, o.Value, o.ValueText} } _, err := tx.CopyFrom(ctx, pgx.Identifier{"dataset_observations"}, - []string{"dataset_id", "kato_code", "date", "value", "value_text"}, + []string{"dataset_id", "kato_code", "date", "date_to", "value", "value_text"}, pgx.CopyFromRows(rows), ) if err != nil { @@ -280,11 +303,11 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u return mapError(tx.Commit(ctx)) } -const observationColumns = `id, dataset_id, kato_code, to_char(date, 'YYYY-MM-DD') AS date, value, value_text` +const observationColumns = `id, dataset_id, kato_code, to_char(date, 'YYYY-MM-DD') AS date, to_char(date_to, 'YYYY-MM-DD') AS date_to, value, value_text` func scanObservation(row pgx.Row) (domain.Observation, error) { var o domain.Observation - err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.Date, &o.Value, &o.ValueText) + err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.Date, &o.DateTo, &o.Value, &o.ValueText) return o, err } diff --git a/internal/service/dataset.go b/internal/service/dataset.go index 80c8d83..f4a8f1c 100644 --- a/internal/service/dataset.go +++ b/internal/service/dataset.go @@ -29,6 +29,8 @@ var ( // ErrCOGNotReady is returned when a raster's Cloud-Optimized GeoTIFF has not // been produced yet. ErrCOGNotReady = errors.New("cog is not ready") + // ErrSLDNotFound is returned when a dataset has no SLD style file uploaded. + ErrSLDNotFound = fmt.Errorf("%w: dataset has no sld", domain.ErrNotFound) ) // DatasetRepository is the persistence behaviour DatasetService needs. @@ -47,6 +49,7 @@ type DatasetRepository interface { SetGeoJSON(ctx context.Context, id uuid.UUID, geojson []byte) error ClearGeometryAndGeoJSON(ctx context.Context, id uuid.UUID) error SaveMapping(ctx context.Context, id uuid.UUID, katoColumn string, years []domain.YearColumn) (domain.Dataset, error) + SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error) ReplaceObservations(ctx context.Context, datasetID uuid.UUID, obs []domain.Observation) error ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error) CountObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string) (int, error) @@ -235,6 +238,82 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat return dataset, nil } +// UploadSLDInput carries an SLD (Styled Layer Descriptor) style file to attach +// to an existing dataset. +type UploadSLDInput struct { + Filename string + ContentType string + Size int64 + Reader io.Reader +} + +// UploadSLD validates and stores an SLD style file for a dataset, replacing +// any previously stored one. Applies to every file_type. +func (s *DatasetService) UploadSLD(ctx context.Context, id uuid.UUID, in UploadSLDInput) (domain.Dataset, error) { + dataset, err := s.repo.GetByID(ctx, id) + if err != nil { + return domain.Dataset{}, err + } + + ext := strings.ToLower(filepath.Ext(in.Filename)) + if !domain.ExtensionAllowedForSLD(ext) { + return domain.Dataset{}, fmt.Errorf("%w: sld file must have one of these extensions: %s", + domain.ErrValidation, strings.Join(domain.AllowedSLDExtensions, ", ")) + } + + head := make([]byte, 512) + n, err := io.ReadFull(in.Reader, head) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return domain.Dataset{}, fmt.Errorf("read sld upload: %w", err) + } + head = head[:n] + if err := domain.ValidateFileContent(ext, head); err != nil { + return domain.Dataset{}, fmt.Errorf("%w: %s", domain.ErrValidation, err) + } + content := io.MultiReader(bytes.NewReader(head), in.Reader) + + key := deriveSLDKey(dataset.StorageKey, in.Filename) + if err := s.store.Put(ctx, key, content, in.Size, in.ContentType); err != nil { + return domain.Dataset{}, err + } + + updated, err := s.repo.SetSLD(ctx, id, key, in.Filename, in.ContentType) + if err != nil { + // Compensate: the row was not updated, so the object would be orphaned. + _ = s.store.Remove(ctx, key) + return domain.Dataset{}, err + } + + if dataset.SldStorageKey != nil && *dataset.SldStorageKey != key { + _ = s.store.Remove(ctx, *dataset.SldStorageKey) // best-effort: drop the replaced file + } + return updated, nil +} + +// DownloadSLD returns the dataset metadata and a reader for its stored SLD +// style file. Fails with ErrSLDNotFound when none has been uploaded. The +// caller must close the reader. +func (s *DatasetService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain.Dataset, io.ReadCloser, error) { + dataset, err := s.repo.GetByID(ctx, id) + if err != nil { + return domain.Dataset{}, nil, err + } + if dataset.SldStorageKey == nil { + return domain.Dataset{}, nil, ErrSLDNotFound + } + obj, err := s.store.Get(ctx, *dataset.SldStorageKey) + if err != nil { + return domain.Dataset{}, nil, err + } + return dataset, obj, nil +} + +// deriveSLDKey places a dataset's SLD style file alongside the original under +// an sld/ prefix, named after the uploaded SLD file. +func deriveSLDKey(storageKey, filename string) string { + return path.Join(path.Dir(storageKey), "sld", filename) +} + // enqueueProcessing schedules the appropriate async job for a dataset based on // its file type: vector_with_kato is parsed for column selection, plain vector // has its attribute table extracted into properties, and raster is converted to @@ -603,9 +682,19 @@ func (s *DatasetService) SaveMapping(ctx context.Context, id uuid.UUID, in Mappi if _, ok := known[yc.Column]; !ok { return domain.Dataset{}, fmt.Errorf("%w: year column %q is not among the detected columns", domain.ErrValidation, yc.Column) } - if _, err := time.Parse("2006-01-02", yc.Date); err != nil { + from, err := time.Parse("2006-01-02", yc.Date) + if err != nil { return domain.Dataset{}, fmt.Errorf("%w: invalid date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.Date, yc.Column) } + if yc.DateTo != nil { + to, err := time.Parse("2006-01-02", *yc.DateTo) + if err != nil { + return domain.Dataset{}, fmt.Errorf("%w: invalid date_to %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, *yc.DateTo, yc.Column) + } + if to.Before(from) { + return domain.Dataset{}, fmt.Errorf("%w: date_to %q is before date %q for column %q", domain.ErrValidation, *yc.DateTo, yc.Date, yc.Column) + } + } } dataset, err = s.repo.SaveMapping(ctx, id, in.KatoColumn, in.YearColumns) @@ -659,8 +748,9 @@ func (s *DatasetService) Extract(ctx context.Context, id uuid.UUID) error { } // buildObservations unpivots rows into observations. Rows without a KATO code -// are skipped; duplicate (kato, date) pairs keep the last value. Numeric cells -// populate Value, others ValueText. +// are skipped; duplicate (kato, date, date_to) tuples keep the last value. +// Numeric cells populate Value, others ValueText. A year column's DateTo (nil +// for a point-in-time mapping) carries through to the observation unchanged. func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.YearColumn, rows []map[string]string) []domain.Observation { obs := make([]domain.Observation, 0, len(rows)*len(years)) index := make(map[string]int) @@ -671,7 +761,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye continue } for _, yc := range years { - o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date} + o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date, DateTo: yc.DateTo} if raw := strings.TrimSpace(row[yc.Column]); raw != "" { if f, err := strconv.ParseFloat(raw, 64); err == nil { o.Value = &f @@ -679,7 +769,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye o.ValueText = &raw } } - key := kato + "\x00" + yc.Date + key := kato + "\x00" + yc.Date + "\x00" + dateToKey(yc.DateTo) if i, ok := index[key]; ok { obs[i] = o } else { @@ -691,6 +781,14 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye return obs } +// dateToKey normalizes an optional period end date for use in a dedup key. +func dateToKey(dateTo *string) string { + if dateTo == nil { + return "" + } + return *dateTo +} + // ObservationPage is a page of observations with pagination metadata. type ObservationPage struct { Items []domain.Observation @@ -789,7 +887,11 @@ func (s *DatasetService) generateGeoJSON(ctx context.Context, id uuid.UUID) erro // joins the districts table on KATO code: one Feature is emitted per KATO, its // boundary taken from the districts table and its observation values nested // under a `data` object (keyed by date) alongside `kato` and `name`. KATO codes -// with no matching district are skipped. +// with no matching district are skipped. When any observation carries a period +// (DateTo set), its end date is additionally exposed under a `periods` object +// (keyed by the same start date used in `data`); `periods` is omitted entirely +// when every observation is point-in-time, so a dataset mapped without periods +// produces byte-identical output to before periods existed. func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.RawMessage, error) { dataset, err := s.repo.GetByID(ctx, id) if err != nil { @@ -816,8 +918,8 @@ func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.R if err != nil { return nil, err } - grouped, order := groupObservationsByKato(obs) - features, err := s.districtFeatures(ctx, grouped, order) + grouped, periods, order := groupObservationsByKato(obs) + features, err := s.districtFeatures(ctx, grouped, periods, order) if err != nil { return nil, err } @@ -850,9 +952,12 @@ func (s *DatasetService) loadGeoJSONDataset(ctx context.Context, id uuid.UUID) ( // districtFeatures builds one Feature per KATO from the districts table, with // the grouped observation values (keyed by date) nested under a `data` object -// alongside `kato` and `name` in each Feature's properties. KATO codes with no +// alongside `kato` and `name` in each Feature's properties. When periods holds +// an entry for the KATO (at least one observation had a period), its date-> +// date_to map is additionally nested under a `periods` object; a KATO with no +// period observations gets no `periods` key at all. KATO codes with no // matching district are skipped. order drives the deterministic feature order. -func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, order []string) ([]domain.Feature, error) { +func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, periods map[string]map[string]string, order []string) ([]domain.Feature, error) { districts, err := s.repo.DistrictGeometriesByKato(ctx, order) if err != nil { return nil, err @@ -863,10 +968,14 @@ func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[strin if !ok { continue // skip KATO codes with no district boundary } + props := map[string]any{"kato": kato, "name": dist.Name, "data": grouped[kato]} + if p, ok := periods[kato]; ok { + props["periods"] = p + } features = append(features, domain.Feature{ Type: domain.GeoJSONFeature, Geometry: dist.Geometry, - Properties: map[string]any{"kato": kato, "name": dist.Name, "data": grouped[kato]}, + Properties: props, }) } return features, nil @@ -909,11 +1018,15 @@ func vectorFeatureProperties(raw json.RawMessage) map[string]any { // groupObservationsByKato groups observations by KATO code into date->value // maps. The value is the numeric Value when present, otherwise ValueText, -// otherwise nil (an empty cell). order lists KATO codes in first-seen order so -// the resulting feature order is deterministic. -func groupObservationsByKato(obs []domain.Observation) (map[string]map[string]any, []string) { - grouped := make(map[string]map[string]any) - order := make([]string, 0) +// otherwise nil (an empty cell). It also groups, per KATO, a date->date_to map +// for observations that carry a period (DateTo set); a KATO with no period +// observations gets no entry in periods at all, so callers can tell "no +// periods" apart from "empty periods". order lists KATO codes in first-seen +// order so the resulting feature order is deterministic. +func groupObservationsByKato(obs []domain.Observation) (grouped map[string]map[string]any, periods map[string]map[string]string, order []string) { + grouped = make(map[string]map[string]any) + periods = make(map[string]map[string]string) + order = make([]string, 0) for _, o := range obs { values, ok := grouped[o.KatoCode] if !ok { @@ -922,8 +1035,17 @@ func groupObservationsByKato(obs []domain.Observation) (map[string]map[string]an order = append(order, o.KatoCode) } values[o.Date] = observationValue(o) + + if o.DateTo != nil { + p, ok := periods[o.KatoCode] + if !ok { + p = make(map[string]string) + periods[o.KatoCode] = p + } + p[o.Date] = *o.DateTo + } } - return grouped, order + return grouped, periods, order } // observationValue returns the typed cell value: numeric Value, else ValueText, diff --git a/internal/service/dataset_test.go b/internal/service/dataset_test.go index 74aba7b..3ed262c 100644 --- a/internal/service/dataset_test.go +++ b/internal/service/dataset_test.go @@ -137,6 +137,18 @@ func (r *stubDatasetRepo) SaveMapping(_ context.Context, id uuid.UUID, kato stri return d, nil } +func (r *stubDatasetRepo) SetSLD(_ context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error) { + d, ok := r.store[id] + if !ok { + return domain.Dataset{}, domain.ErrNotFound + } + d.SldStorageKey = &key + d.SldFilename = &filename + d.SldContentType = &contentType + r.store[id] = d + return d, nil +} + func (r *stubDatasetRepo) MarkReady(_ context.Context, id uuid.UUID, geometry []byte) error { d, ok := r.store[id] if !ok { @@ -776,6 +788,39 @@ func TestDatasetService_SaveMapping(t *testing.T) { } }) + t.Run("bad date_to", func(t *testing.T) { + _, svc := newSvc() + _, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-01-01", DateTo: strPtr("2023")}}}) + if !errors.Is(err, domain.ErrValidation) { + t.Fatalf("want ErrValidation, got %v", err) + } + }) + + t.Run("date_to before date", func(t *testing.T) { + _, svc := newSvc() + _, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-12-31", DateTo: strPtr("2023-01-01")}}}) + if !errors.Is(err, domain.ErrValidation) { + t.Fatalf("want ErrValidation, got %v", err) + } + }) + + t.Run("period mapping succeeds", func(t *testing.T) { + repo, svc := newSvc() + got, err := svc.SaveMapping(ctx, id, MappingInput{ + KatoColumn: "като", + YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2020-01-01", DateTo: strPtr("2023-12-31")}}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got.YearColumns) != 1 || got.YearColumns[0].DateTo == nil || *got.YearColumns[0].DateTo != "2023-12-31" { + t.Fatalf("period not saved: %+v", got.YearColumns) + } + if len(repo.store[id].YearColumns) != 1 { + t.Fatalf("period not persisted: %+v", repo.store[id].YearColumns) + } + }) + t.Run("no year columns", func(t *testing.T) { _, svc := newSvc() _, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като"}) @@ -845,6 +890,45 @@ func TestBuildObservations(t *testing.T) { if o := byKey["751010000|2025-01-01"]; o.ValueText == nil || *o.ValueText != "n/a" { t.Errorf("non-numeric cell not stored as value_text: %+v", o) } + for _, o := range obs { + if o.DateTo != nil { + t.Fatalf("point-in-time mapping must not set DateTo, got %+v", o) + } + } +} + +func TestBuildObservations_Period(t *testing.T) { + id := uuid.New() + years := []domain.YearColumn{ + {Column: "F_2020_2023", Date: "2020-01-01", DateTo: strPtr("2023-12-31")}, // period + {Column: "F_2024", Date: "2024-01-01"}, // point-in-time, unaffected + } + rows := []map[string]string{ + {"като": "751010000", "F_2020_2023": "100", "F_2024": "200"}, + } + + obs := buildObservations(id, "като", years, rows) + if len(obs) != 2 { + t.Fatalf("want 2 observations, got %d", len(obs)) + } + + byDate := map[string]domain.Observation{} + for _, o := range obs { + byDate[o.Date] = o + } + + period := byDate["2020-01-01"] + if period.DateTo == nil || *period.DateTo != "2023-12-31" { + t.Fatalf("period observation missing date_to: %+v", period) + } + if period.Value == nil || *period.Value != 100 { + t.Fatalf("period observation value wrong: %+v", period) + } + + point := byDate["2024-01-01"] + if point.DateTo != nil { + t.Fatalf("point-in-time observation must have nil date_to (backward compat), got %+v", point) + } } func TestDatasetService_Extract(t *testing.T) { @@ -959,6 +1043,68 @@ func TestDatasetService_GeoJSON_DistrictJoin(t *testing.T) { } } +func TestDatasetService_GeoJSON_Period(t *testing.T) { + id := uuid.New() + repo := newStubDatasetRepo() + repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeVectorWithKato, Status: domain.DatasetStatusReady} + + period, point := 100.0, 200.0 + repo.observations[id] = []domain.Observation{ + {KatoCode: "710000000", Date: "2020-01-01", DateTo: strPtr("2023-12-31"), Value: &period}, + {KatoCode: "710000000", Date: "2024-01-01", Value: &point}, // point-in-time, no period + } + repo.districts["710000000"] = domain.District{ + Kato: "710000000", Name: "Astana", + Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`), + } + + svc := newDatasetService(repo, &stubStore{}, true) + fc := mustGeoJSON(t, svc, id) + if len(fc.Features) != 1 { + t.Fatalf("want 1 feature, got %d", len(fc.Features)) + } + f := fc.Features[0] + + data, ok := f.Properties["data"].(map[string]any) + if !ok { + t.Fatalf("observations not nested under data: %+v", f.Properties) + } + if data["2020-01-01"] != 100.0 || data["2024-01-01"] != 200.0 { + t.Fatalf("unexpected data: %+v", data) + } + + periods, ok := f.Properties["periods"].(map[string]any) + if !ok { + t.Fatalf("periods not exposed as a map: %+v", f.Properties) + } + if len(periods) != 1 || periods["2020-01-01"] != "2023-12-31" { + t.Fatalf("period end date not exposed: %+v", periods) + } + if _, has := periods["2024-01-01"]; has { + t.Fatalf("point-in-time date must not appear in periods: %+v", periods) + } +} + +func TestDatasetService_GeoJSON_NoPeriods_OmitsPeriodsKey(t *testing.T) { + // Regression: a dataset mapped without any period must produce output + // byte-identical to before periods existed - no "periods" key at all. + id := uuid.New() + repo := newStubDatasetRepo() + repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeVectorWithKato, Status: domain.DatasetStatusReady} + v := 100.0 + repo.observations[id] = []domain.Observation{{KatoCode: "710000000", Date: "2020-01-01", Value: &v}} + repo.districts["710000000"] = domain.District{ + Kato: "710000000", Name: "Astana", + Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`), + } + + svc := newDatasetService(repo, &stubStore{}, true) + fc := mustGeoJSON(t, svc, id) + if _, has := fc.Features[0].Properties["periods"]; has { + t.Fatalf("periods key must be absent when no observation has a period: %+v", fc.Features[0].Properties) + } +} + func TestDatasetService_GeoJSON_IgnoresDatasetGeometry(t *testing.T) { id := uuid.New() repo := newStubDatasetRepo() @@ -1294,6 +1440,125 @@ func TestDatasetService_ReprocessAll_FiltersByFileType(t *testing.T) { } } +func validSLDUpload() UploadSLDInput { + body := `` + return UploadSLDInput{ + Filename: "style.sld", + ContentType: "application/xml", + Size: int64(len(body)), + Reader: strings.NewReader(body), + } +} + +func TestDatasetService_UploadSLD_Success(t *testing.T) { + id := uuid.New() + repo := newStubDatasetRepo() + repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeRaster, StorageKey: "uid/dem.tif"} + store := &stubStore{} + svc := newDatasetService(repo, store, true) + + got, err := svc.UploadSLD(context.Background(), id, validSLDUpload()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.SldStorageKey == nil || *got.SldStorageKey != "uid/sld/style.sld" { + t.Fatalf("unexpected sld storage key: %v", got.SldStorageKey) + } + if got.SldFilename == nil || *got.SldFilename != "style.sld" { + t.Fatalf("sld filename not stored: %v", got.SldFilename) + } + if len(store.put) != 1 || store.put[0] != "uid/sld/style.sld" { + t.Fatalf("sld not uploaded to store: %v", store.put) + } +} + +func TestDatasetService_UploadSLD_ReplacesPrevious(t *testing.T) { + id := uuid.New() + repo := newStubDatasetRepo() + oldKey := "uid/sld/old.sld" + repo.store[id] = domain.Dataset{ + ID: id, FileType: domain.FileTypeVector, StorageKey: "uid/data.geojson", + SldStorageKey: &oldKey, + } + store := &stubStore{} + svc := newDatasetService(repo, store, true) + + in := validSLDUpload() + in.Filename = "new.sld" + got, err := svc.UploadSLD(context.Background(), id, in) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.SldStorageKey == nil || *got.SldStorageKey != "uid/sld/new.sld" { + t.Fatalf("unexpected sld storage key: %v", got.SldStorageKey) + } + if len(store.removed) != 1 || store.removed[0] != oldKey { + t.Fatalf("old sld object not removed: %v", store.removed) + } +} + +func TestDatasetService_UploadSLD_RejectsBadExtension(t *testing.T) { + id := uuid.New() + repo := newStubDatasetRepo() + repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"} + store := &stubStore{} + svc := newDatasetService(repo, store, true) + + in := validSLDUpload() + in.Filename = "style.zip" + _, err := svc.UploadSLD(context.Background(), id, in) + if !errors.Is(err, domain.ErrValidation) { + t.Fatalf("want ErrValidation, got %v", err) + } + if len(store.put) != 0 { + t.Fatalf("nothing should be uploaded on validation failure, got %v", store.put) + } +} + +func TestDatasetService_UploadSLD_RejectsMismatchedContent(t *testing.T) { + id := uuid.New() + repo := newStubDatasetRepo() + repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"} + store := &stubStore{} + svc := newDatasetService(repo, store, true) + + in := validSLDUpload() + in.Reader = strings.NewReader(`{"not":"xml"}`) + _, err := svc.UploadSLD(context.Background(), id, in) + if !errors.Is(err, domain.ErrValidation) { + t.Fatalf("want ErrValidation, got %v", err) + } +} + +func TestDatasetService_DownloadSLD(t *testing.T) { + id := uuid.New() + repo := newStubDatasetRepo() + repo.store[id] = domain.Dataset{ID: id, StorageKey: "uid/data.geojson"} + store := &stubStore{} + svc := newDatasetService(repo, store, true) + + t.Run("not found before upload", func(t *testing.T) { + _, _, err := svc.DownloadSLD(context.Background(), id) + if !errors.Is(err, ErrSLDNotFound) { + t.Fatalf("want ErrSLDNotFound, got %v", err) + } + }) + + t.Run("streams after upload", func(t *testing.T) { + if _, err := svc.UploadSLD(context.Background(), id, validSLDUpload()); err != nil { + t.Fatalf("upload: %v", err) + } + _, obj, err := svc.DownloadSLD(context.Background(), id) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer obj.Close() + if _, err := io.ReadAll(obj); err != nil { + t.Fatalf("read: %v", err) + } + }) +} + func TestDatasetService_Delete_RemovesObject(t *testing.T) { repo := newStubDatasetRepo() id := uuid.New() diff --git a/internal/transport/http/dataset_handler.go b/internal/transport/http/dataset_handler.go index db3949a..8e3563c 100644 --- a/internal/transport/http/dataset_handler.go +++ b/internal/transport/http/dataset_handler.go @@ -43,6 +43,8 @@ func (h *DatasetHandler) Register(r chi.Router) { r.Get("/{id}.cog", h.cog) r.Get("/{id}/status", h.status) r.Get("/{id}/download", h.download) + r.Post("/{id}/sld", h.uploadSLD) + r.Get("/{id}/sld", h.downloadSLD) r.Post("/{id}/mapping", h.mapping) r.Get("/{id}/observations", h.observations) r.Delete("/{id}", h.delete) @@ -78,8 +80,9 @@ func (h *DatasetHandler) status(w http.ResponseWriter, r *http.Request) { } type yearColumnInput struct { - Column string `json:"column" validate:"required"` - Date string `json:"date" validate:"required,datetime=2006-01-02"` + Column string `json:"column" validate:"required"` + Date string `json:"date" validate:"required,datetime=2006-01-02"` + DateTo *string `json:"date_to" validate:"omitempty,datetime=2006-01-02"` } type mappingRequest struct { @@ -106,7 +109,7 @@ func (h *DatasetHandler) mapping(w http.ResponseWriter, r *http.Request) { in := service.MappingInput{KatoColumn: req.KatoColumn} for _, yc := range req.YearColumns { - in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, Date: yc.Date}) + in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, Date: yc.Date, DateTo: yc.DateTo}) } dataset, err := h.svc.SaveMapping(r.Context(), id, in) @@ -533,6 +536,74 @@ func (h *DatasetHandler) download(w http.ResponseWriter, r *http.Request) { } } +// uploadSLD stores (or replaces) a dataset's SLD (Styled Layer Descriptor) +// style file. Multipart upload, mirroring POST /datasets. Applies to every +// file_type. +func (h *DatasetHandler) uploadSLD(w http.ResponseWriter, r *http.Request) { + id, ok := parseUUIDParam(w, r, "id") + if !ok { + return + } + if err := r.ParseMultipartForm(maxUploadBytes); err != nil { + httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data") + return + } + + file, header, err := r.FormFile("file") + if err != nil { + httputil.WriteError(w, http.StatusBadRequest, "file is required") + return + } + defer file.Close() + + contentType := header.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/xml" + } + + dataset, err := h.svc.UploadSLD(r.Context(), id, service.UploadSLDInput{ + Filename: header.Filename, + ContentType: contentType, + Size: header.Size, + Reader: file, + }) + if err != nil { + respondDomainError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, dataset) +} + +// downloadSLD streams a dataset's stored SLD style file. Returns 404 if none +// has been uploaded yet. +func (h *DatasetHandler) downloadSLD(w http.ResponseWriter, r *http.Request) { + id, ok := parseUUIDParam(w, r, "id") + if !ok { + return + } + dataset, obj, err := h.svc.DownloadSLD(r.Context(), id) + if err != nil { + respondDomainError(w, err) + return + } + defer obj.Close() + + contentType := "application/xml" + if dataset.SldContentType != nil && *dataset.SldContentType != "" { + contentType = *dataset.SldContentType + } + filename := "style.sld" + if dataset.SldFilename != nil && *dataset.SldFilename != "" { + filename = *dataset.SldFilename + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`) + if _, err := io.Copy(w, obj); err != nil { + // Headers are already sent; nothing useful to return to the client. + return + } +} + func (h *DatasetHandler) delete(w http.ResponseWriter, r *http.Request) { id, ok := parseUUIDParam(w, r, "id") if !ok { diff --git a/migrations/00008_add_sld_to_datasets.sql b/migrations/00008_add_sld_to_datasets.sql new file mode 100644 index 0000000..a961652 --- /dev/null +++ b/migrations/00008_add_sld_to_datasets.sql @@ -0,0 +1,12 @@ +-- +goose Up +-- Optional SLD (Styled Layer Descriptor) style file for a dataset. Applies to +-- every file_type; a dataset may have at most one (uploading again replaces +-- it). +ALTER TABLE datasets ADD COLUMN sld_storage_key TEXT; +ALTER TABLE datasets ADD COLUMN sld_filename VARCHAR(255); +ALTER TABLE datasets ADD COLUMN sld_content_type VARCHAR(255); + +-- +goose Down +ALTER TABLE datasets DROP COLUMN sld_storage_key; +ALTER TABLE datasets DROP COLUMN sld_filename; +ALTER TABLE datasets DROP COLUMN sld_content_type; diff --git a/migrations/00009_add_period_to_dataset_observations.sql b/migrations/00009_add_period_to_dataset_observations.sql new file mode 100644 index 0000000..76ff0b9 --- /dev/null +++ b/migrations/00009_add_period_to_dataset_observations.sql @@ -0,0 +1,15 @@ +-- +goose Up +-- Optional period end date, so a year_column mapping can represent a date +-- range (date..date_to) instead of a single point in time. NULL preserves the +-- pre-existing point-in-time behavior; the unique index treats a NULL date_to +-- as equal to date itself so point observations keep deduplicating exactly as +-- before. +ALTER TABLE dataset_observations ADD COLUMN date_to DATE; + +DROP INDEX uq_dataset_observations; +CREATE UNIQUE INDEX uq_dataset_observations ON dataset_observations (dataset_id, kato_code, date, COALESCE(date_to, date)); + +-- +goose Down +DROP INDEX uq_dataset_observations; +ALTER TABLE dataset_observations DROP COLUMN date_to; +CREATE UNIQUE INDEX uq_dataset_observations ON dataset_observations (dataset_id, kato_code, date);