infureal/gis
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add SLD endpoints, add date_to to vector_with_kato

Browse Source
This commit is contained in:
Bakhtiyar Issakhmetov
2026-08-01 22:09:59 +05:00
parent 00a0cd1b8e
commit 1e0844ca82
8 changed files with 656 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
internal/service/dataset.go
View File

@@ -29,6 +29,8 @@ var (
// ErrCOGNotReady is returned when a raster's Cloud-Optimized GeoTIFF has not
// been produced yet.
ErrCOGNotReady = errors.New("cog is not ready")
// ErrSLDNotFound is returned when a dataset has no SLD style file uploaded.
ErrSLDNotFound = fmt.Errorf("%w: dataset has no sld", domain.ErrNotFound)
)
// DatasetRepository is the persistence behaviour DatasetService needs.
@@ -47,6 +49,7 @@ type DatasetRepository interface {
SetGeoJSON(ctx context.Context, id uuid.UUID, geojson []byte) error
ClearGeometryAndGeoJSON(ctx context.Context, id uuid.UUID) error
SaveMapping(ctx context.Context, id uuid.UUID, katoColumn string, years []domain.YearColumn) (domain.Dataset, error)
SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.Dataset, error)
ReplaceObservations(ctx context.Context, datasetID uuid.UUID, obs []domain.Observation) error
ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error)
CountObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string) (int, error)
@@ -235,6 +238,82 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat
return dataset, nil
}
// UploadSLDInput carries an SLD (Styled Layer Descriptor) style file to attach
// to an existing dataset.
type UploadSLDInput struct {
Filename string
ContentType string
Size int64
Reader io.Reader
}
// UploadSLD validates and stores an SLD style file for a dataset, replacing
// any previously stored one. Applies to every file_type.
func (s *DatasetService) UploadSLD(ctx context.Context, id uuid.UUID, in UploadSLDInput) (domain.Dataset, error) {
dataset, err := s.repo.GetByID(ctx, id)
if err != nil {
return domain.Dataset{}, err
}
ext := strings.ToLower(filepath.Ext(in.Filename))
if !domain.ExtensionAllowedForSLD(ext) {
return domain.Dataset{}, fmt.Errorf("%w: sld file must have one of these extensions: %s",
domain.ErrValidation, strings.Join(domain.AllowedSLDExtensions, ", "))
}
head := make([]byte, 512)
n, err := io.ReadFull(in.Reader, head)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return domain.Dataset{}, fmt.Errorf("read sld upload: %w", err)
}
head = head[:n]
if err := domain.ValidateFileContent(ext, head); err != nil {
return domain.Dataset{}, fmt.Errorf("%w: %s", domain.ErrValidation, err)
}
content := io.MultiReader(bytes.NewReader(head), in.Reader)
key := deriveSLDKey(dataset.StorageKey, in.Filename)
if err := s.store.Put(ctx, key, content, in.Size, in.ContentType); err != nil {
return domain.Dataset{}, err
}
updated, err := s.repo.SetSLD(ctx, id, key, in.Filename, in.ContentType)
if err != nil {
// Compensate: the row was not updated, so the object would be orphaned.
_ = s.store.Remove(ctx, key)
return domain.Dataset{}, err
}
if dataset.SldStorageKey != nil && *dataset.SldStorageKey != key {
_ = s.store.Remove(ctx, *dataset.SldStorageKey) // best-effort: drop the replaced file
}
return updated, nil
}
// DownloadSLD returns the dataset metadata and a reader for its stored SLD
// style file. Fails with ErrSLDNotFound when none has been uploaded. The
// caller must close the reader.
func (s *DatasetService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain.Dataset, io.ReadCloser, error) {
dataset, err := s.repo.GetByID(ctx, id)
if err != nil {
return domain.Dataset{}, nil, err
}
if dataset.SldStorageKey == nil {
return domain.Dataset{}, nil, ErrSLDNotFound
}
obj, err := s.store.Get(ctx, *dataset.SldStorageKey)
if err != nil {
return domain.Dataset{}, nil, err
}
return dataset, obj, nil
}
// deriveSLDKey places a dataset's SLD style file alongside the original under
// an sld/ prefix, named after the uploaded SLD file.
func deriveSLDKey(storageKey, filename string) string {
return path.Join(path.Dir(storageKey), "sld", filename)
}
// enqueueProcessing schedules the appropriate async job for a dataset based on
// its file type: vector_with_kato is parsed for column selection, plain vector
// has its attribute table extracted into properties, and raster is converted to
@@ -603,9 +682,19 @@ func (s *DatasetService) SaveMapping(ctx context.Context, id uuid.UUID, in Mappi
if _, ok := known[yc.Column]; !ok {
return domain.Dataset{}, fmt.Errorf("%w: year column %q is not among the detected columns", domain.ErrValidation, yc.Column)
}
if _, err := time.Parse("2006-01-02", yc.Date); err != nil {
from, err := time.Parse("2006-01-02", yc.Date)
if err != nil {
return domain.Dataset{}, fmt.Errorf("%w: invalid date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.Date, yc.Column)
}
if yc.DateTo != nil {
to, err := time.Parse("2006-01-02", *yc.DateTo)
if err != nil {
return domain.Dataset{}, fmt.Errorf("%w: invalid date_to %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, *yc.DateTo, yc.Column)
}
if to.Before(from) {
return domain.Dataset{}, fmt.Errorf("%w: date_to %q is before date %q for column %q", domain.ErrValidation, *yc.DateTo, yc.Date, yc.Column)
}
}
}
dataset, err = s.repo.SaveMapping(ctx, id, in.KatoColumn, in.YearColumns)
@@ -659,8 +748,9 @@ func (s *DatasetService) Extract(ctx context.Context, id uuid.UUID) error {
}
// buildObservations unpivots rows into observations. Rows without a KATO code
// are skipped; duplicate (kato, date) pairs keep the last value. Numeric cells
// populate Value, others ValueText.
// are skipped; duplicate (kato, date, date_to) tuples keep the last value.
// Numeric cells populate Value, others ValueText. A year column's DateTo (nil
// for a point-in-time mapping) carries through to the observation unchanged.
func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.YearColumn, rows []map[string]string) []domain.Observation {
obs := make([]domain.Observation, 0, len(rows)*len(years))
index := make(map[string]int)
@@ -671,7 +761,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
continue
}
for _, yc := range years {
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date}
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date, DateTo: yc.DateTo}
if raw := strings.TrimSpace(row[yc.Column]); raw != "" {
if f, err := strconv.ParseFloat(raw, 64); err == nil {
o.Value = &f
@@ -679,7 +769,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
o.ValueText = &raw
}
}
key := kato + "\x00" + yc.Date
key := kato + "\x00" + yc.Date + "\x00" + dateToKey(yc.DateTo)
if i, ok := index[key]; ok {
obs[i] = o
} else {
@@ -691,6 +781,14 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
return obs
}
// dateToKey normalizes an optional period end date for use in a dedup key.
func dateToKey(dateTo *string) string {
if dateTo == nil {
return ""
}
return *dateTo
}
// ObservationPage is a page of observations with pagination metadata.
type ObservationPage struct {
Items []domain.Observation
@@ -789,7 +887,11 @@ func (s *DatasetService) generateGeoJSON(ctx context.Context, id uuid.UUID) erro
// joins the districts table on KATO code: one Feature is emitted per KATO, its
// boundary taken from the districts table and its observation values nested
// under a `data` object (keyed by date) alongside `kato` and `name`. KATO codes
// with no matching district are skipped.
// with no matching district are skipped. When any observation carries a period
// (DateTo set), its end date is additionally exposed under a `periods` object
// (keyed by the same start date used in `data`); `periods` is omitted entirely
// when every observation is point-in-time, so a dataset mapped without periods
// produces byte-identical output to before periods existed.
func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.RawMessage, error) {
dataset, err := s.repo.GetByID(ctx, id)
if err != nil {
@@ -816,8 +918,8 @@ func (s *DatasetService) buildGeoJSON(ctx context.Context, id uuid.UUID) (json.R
if err != nil {
return nil, err
}
grouped, order := groupObservationsByKato(obs)
features, err := s.districtFeatures(ctx, grouped, order)
grouped, periods, order := groupObservationsByKato(obs)
features, err := s.districtFeatures(ctx, grouped, periods, order)
if err != nil {
return nil, err
}
@@ -850,9 +952,12 @@ func (s *DatasetService) loadGeoJSONDataset(ctx context.Context, id uuid.UUID) (
// districtFeatures builds one Feature per KATO from the districts table, with
// the grouped observation values (keyed by date) nested under a `data` object
// alongside `kato` and `name` in each Feature's properties. KATO codes with no
// alongside `kato` and `name` in each Feature's properties. When periods holds
// an entry for the KATO (at least one observation had a period), its date->
// date_to map is additionally nested under a `periods` object; a KATO with no
// period observations gets no `periods` key at all. KATO codes with no
// matching district are skipped. order drives the deterministic feature order.
func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, order []string) ([]domain.Feature, error) {
func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, periods map[string]map[string]string, order []string) ([]domain.Feature, error) {
districts, err := s.repo.DistrictGeometriesByKato(ctx, order)
if err != nil {
return nil, err
@@ -863,10 +968,14 @@ func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[strin
if !ok {
continue // skip KATO codes with no district boundary
}
props := map[string]any{"kato": kato, "name": dist.Name, "data": grouped[kato]}
if p, ok := periods[kato]; ok {
props["periods"] = p
}
features = append(features, domain.Feature{
Type: domain.GeoJSONFeature,
Geometry: dist.Geometry,
Properties: map[string]any{"kato": kato, "name": dist.Name, "data": grouped[kato]},
Properties: props,
})
}
return features, nil
@@ -909,11 +1018,15 @@ func vectorFeatureProperties(raw json.RawMessage) map[string]any {
// groupObservationsByKato groups observations by KATO code into date->value
// maps. The value is the numeric Value when present, otherwise ValueText,
// otherwise nil (an empty cell). order lists KATO codes in first-seen order so
// the resulting feature order is deterministic.
func groupObservationsByKato(obs []domain.Observation) (map[string]map[string]any, []string) {
grouped := make(map[string]map[string]any)
order := make([]string, 0)
// otherwise nil (an empty cell). It also groups, per KATO, a date->date_to map
// for observations that carry a period (DateTo set); a KATO with no period
// observations gets no entry in periods at all, so callers can tell "no
// periods" apart from "empty periods". order lists KATO codes in first-seen
// order so the resulting feature order is deterministic.
func groupObservationsByKato(obs []domain.Observation) (grouped map[string]map[string]any, periods map[string]map[string]string, order []string) {
grouped = make(map[string]map[string]any)
periods = make(map[string]map[string]string)
order = make([]string, 0)
for _, o := range obs {
values, ok := grouped[o.KatoCode]
if !ok {
@@ -922,8 +1035,17 @@ func groupObservationsByKato(obs []domain.Observation) (map[string]map[string]an
order = append(order, o.KatoCode)
}
values[o.Date] = observationValue(o)
if o.DateTo != nil {
p, ok := periods[o.KatoCode]
if !ok {
p = make(map[string]string)
periods[o.KatoCode] = p
}
p[o.Date] = *o.DateTo
}
}
return grouped, order
return grouped, periods, order
}
// observationValue returns the typed cell value: numeric Value, else ValueText,