infureal/gis
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(datasets): periods (start_date/end_date), dg_id column, date filters; rename observation dates

Browse Source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Bakhtiyar Issakhmetov
2026-09-05 13:16:07 +05:00
parent 37d4f3519b
commit 922812b765
10 changed files with 729 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
internal/service/dataset.go
View File

@@ -37,7 +37,8 @@ var (
type DatasetRepository interface {
Create(ctx context.Context, d domain.Dataset) (domain.Dataset, error)
GetByID(ctx context.Context, id uuid.UUID) (domain.Dataset, error)
UpdateFields(ctx context.Context, id, categoryID uuid.UUID, name string, description, unit *string) (domain.Dataset, error)
UpdateFields(ctx context.Context, id uuid.UUID, f domain.DatasetFields) (domain.Dataset, error)
LatestInGroup(ctx context.Context, groupID uuid.UUID, status *string, fileType *domain.FileType) (domain.Dataset, error)
ListSummaries(ctx context.Context, filter domain.DatasetFilter, limit, offset int) ([]domain.DatasetSummary, error)
Count(ctx context.Context, filter domain.DatasetFilter) (int, error)
Delete(ctx context.Context, id uuid.UUID) error
@@ -125,6 +126,11 @@ type UploadInput struct {
ContentType string
Size int64
Reader io.Reader
// DatasetGroupID optionally files the dataset under a dataset group.
DatasetGroupID *uuid.UUID
// StartDate / EndDate is the period the dataset describes (YYYY-MM-DD).
StartDate *string
EndDate *string
}
// DatasetService implements dataset business rules and object storage handling.
@@ -190,6 +196,9 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat
if err := s.ensureCategoryExists(ctx, in.CategoryID); err != nil {
return domain.Dataset{}, err
}
if err := domain.ValidateDatePeriod(in.StartDate, in.EndDate); err != nil {
return domain.Dataset{}, err
}
name := in.Name
if name == "" {
@@ -210,18 +219,21 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat
}
dataset, err := s.repo.Create(ctx, domain.Dataset{
CategoryID: in.CategoryID,
Name: name,
Description: in.Description,
Unit: in.Unit,
Meta: in.Meta,
Automated: in.Automated,
Status: status,
Filename: in.Filename,
StorageKey: storageKey,
FileType: in.FileType,
SizeBytes: in.Size,
ContentType: in.ContentType,
CategoryID: in.CategoryID,
DatasetGroupID: in.DatasetGroupID,
StartDate: in.StartDate,
EndDate: in.EndDate,
Name: name,
Description: in.Description,
Unit: in.Unit,
Meta: in.Meta,
Automated: in.Automated,
Status: status,
Filename: in.Filename,
StorageKey: storageKey,
FileType: in.FileType,
SizeBytes: in.Size,
ContentType: in.ContentType,
})
if err != nil {
// Compensate: the row was not written, so the object would be orphaned.
@@ -682,17 +694,17 @@ func (s *DatasetService) SaveMapping(ctx context.Context, id uuid.UUID, in Mappi
if _, ok := known[yc.Column]; !ok {
return domain.Dataset{}, fmt.Errorf("%w: year column %q is not among the detected columns", domain.ErrValidation, yc.Column)
}
from, err := time.Parse("2006-01-02", yc.Date)
from, err := time.Parse("2006-01-02", yc.StartDate)
if err != nil {
return domain.Dataset{}, fmt.Errorf("%w: invalid date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.Date, yc.Column)
return domain.Dataset{}, fmt.Errorf("%w: invalid start_date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.StartDate, yc.Column)
}
if yc.DateTo != nil {
to, err := time.Parse("2006-01-02", *yc.DateTo)
if yc.EndDate != nil {
to, err := time.Parse("2006-01-02", *yc.EndDate)
if err != nil {
return domain.Dataset{}, fmt.Errorf("%w: invalid date_to %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, *yc.DateTo, yc.Column)
return domain.Dataset{}, fmt.Errorf("%w: invalid end_date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, *yc.EndDate, yc.Column)
}
if to.Before(from) {
return domain.Dataset{}, fmt.Errorf("%w: date_to %q is before date %q for column %q", domain.ErrValidation, *yc.DateTo, yc.Date, yc.Column)
return domain.Dataset{}, fmt.Errorf("%w: end_date %q is before start_date %q for column %q", domain.ErrValidation, *yc.EndDate, yc.StartDate, yc.Column)
}
}
}
@@ -748,9 +760,10 @@ func (s *DatasetService) Extract(ctx context.Context, id uuid.UUID) error {
}
// buildObservations unpivots rows into observations. Rows without a KATO code
// are skipped; duplicate (kato, date, date_to) tuples keep the last value.
// Numeric cells populate Value, others ValueText. A year column's DateTo (nil
// for a point-in-time mapping) carries through to the observation unchanged.
// are skipped; duplicate (kato, start_date, end_date) tuples keep the last
// value. Numeric cells populate Value, others ValueText. A year column's
// EndDate (nil for a point-in-time mapping) carries through to the observation
// unchanged.
func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.YearColumn, rows []map[string]string) []domain.Observation {
obs := make([]domain.Observation, 0, len(rows)*len(years))
index := make(map[string]int)
@@ -761,7 +774,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
continue
}
for _, yc := range years {
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date, DateTo: yc.DateTo}
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, StartDate: yc.StartDate, EndDate: yc.EndDate}
if raw := strings.TrimSpace(row[yc.Column]); raw != "" {
if f, err := strconv.ParseFloat(raw, 64); err == nil {
o.Value = &f
@@ -769,7 +782,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
o.ValueText = &raw
}
}
key := kato + "\x00" + yc.Date + "\x00" + dateToKey(yc.DateTo)
key := kato + "\x00" + yc.StartDate + "\x00" + endDateKey(yc.EndDate)
if i, ok := index[key]; ok {
obs[i] = o
} else {
@@ -781,12 +794,12 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
return obs
}
// dateToKey normalizes an optional period end date for use in a dedup key.
func dateToKey(dateTo *string) string {
if dateTo == nil {
// endDateKey normalizes an optional period end date for use in a dedup key.
func endDateKey(endDate *string) string {
if endDate == nil {
return ""
}
return *dateTo
return *endDate
}
// ObservationPage is a page of observations with pagination metadata.
@@ -888,7 +901,7 @@ func (s *DatasetService) generateGeoJSON(ctx context.Context, id uuid.UUID) erro
// boundary taken from the districts table and its observation values nested
// under a `data` object (keyed by date) alongside `kato` and `name`. KATO codes
// with no matching district are skipped. When any observation carries a period
// (DateTo set), its end date is additionally exposed under a `periods` object
// (EndDate set), its end date is additionally exposed under a `periods` object
// (keyed by the same start date used in `data`); `periods` is omitted entirely
// when every observation is point-in-time, so a dataset mapped without periods
// produces byte-identical output to before periods existed.
@@ -954,7 +967,7 @@ func (s *DatasetService) loadGeoJSONDataset(ctx context.Context, id uuid.UUID) (
// the grouped observation values (keyed by date) nested under a `data` object
// alongside `kato` and `name` in each Feature's properties. When periods holds
// an entry for the KATO (at least one observation had a period), its date->
// date_to map is additionally nested under a `periods` object; a KATO with no
// end_date map is additionally nested under a `periods` object; a KATO with no
// period observations gets no `periods` key at all. KATO codes with no
// matching district are skipped. order drives the deterministic feature order.
func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, periods map[string]map[string]string, order []string) ([]domain.Feature, error) {
@@ -1018,8 +1031,8 @@ func vectorFeatureProperties(raw json.RawMessage) map[string]any {
// groupObservationsByKato groups observations by KATO code into date->value
// maps. The value is the numeric Value when present, otherwise ValueText,
// otherwise nil (an empty cell). It also groups, per KATO, a date->date_to map
// for observations that carry a period (DateTo set); a KATO with no period
// otherwise nil (an empty cell). It also groups, per KATO, a start_date->end_date map
// for observations that carry a period (EndDate set); a KATO with no period
// observations gets no entry in periods at all, so callers can tell "no
// periods" apart from "empty periods". order lists KATO codes in first-seen
// order so the resulting feature order is deterministic.
@@ -1034,15 +1047,15 @@ func groupObservationsByKato(obs []domain.Observation) (grouped map[string]map[s
grouped[o.KatoCode] = values
order = append(order, o.KatoCode)
}
values[o.Date] = observationValue(o)
values[o.StartDate] = observationValue(o)
if o.DateTo != nil {
if o.EndDate != nil {
p, ok := periods[o.KatoCode]
if !ok {
p = make(map[string]string)
periods[o.KatoCode] = p
}
p[o.Date] = *o.DateTo
p[o.StartDate] = *o.EndDate
}
}
return grouped, periods, order
@@ -1074,6 +1087,10 @@ type DatasetUpdateInput struct {
Name string
Description *string
Unit *string
// DatasetGroupID, StartDate and EndDate are nullable: nil clears them.
DatasetGroupID *uuid.UUID
StartDate *string
EndDate *string
}
// Update replaces a dataset's editable metadata (category, name, description,
@@ -1082,7 +1099,18 @@ func (s *DatasetService) Update(ctx context.Context, id uuid.UUID, in DatasetUpd
if err := s.ensureCategoryExists(ctx, in.CategoryID); err != nil {
return domain.Dataset{}, err
}
return s.repo.UpdateFields(ctx, id, in.CategoryID, in.Name, in.Description, in.Unit)
if err := domain.ValidateDatePeriod(in.StartDate, in.EndDate); err != nil {
return domain.Dataset{}, err
}
return s.repo.UpdateFields(ctx, id, domain.DatasetFields{
CategoryID: in.CategoryID,
Name: in.Name,
Description: in.Description,
Unit: in.Unit,
DatasetGroupID: in.DatasetGroupID,
StartDate: in.StartDate,
EndDate: in.EndDate,
})
}
// DatasetPatchInput carries a sparse set of metadata field changes. A nil
@@ -1096,6 +1124,13 @@ type DatasetPatchInput struct {
DescriptionSet bool
Unit *string
UnitSet bool
DatasetGroupID *uuid.UUID
DatasetGroupIDSet bool
StartDate *string
StartDateSet bool
EndDate *string
EndDateSet bool
}
// Patch applies a sparse update to a dataset's editable metadata (category,
@@ -1125,8 +1160,31 @@ func (s *DatasetService) Patch(ctx context.Context, id uuid.UUID, in DatasetPatc
if in.UnitSet {
unit = in.Unit
}
groupID := current.DatasetGroupID
if in.DatasetGroupIDSet {
groupID = in.DatasetGroupID
}
startDate := current.StartDate
if in.StartDateSet {
startDate = in.StartDate
}
endDate := current.EndDate
if in.EndDateSet {
endDate = in.EndDate
}
if err := domain.ValidateDatePeriod(startDate, endDate); err != nil {
return domain.Dataset{}, err
}
return s.repo.UpdateFields(ctx, id, categoryID, name, description, unit)
return s.repo.UpdateFields(ctx, id, domain.DatasetFields{
CategoryID: categoryID,
Name: name,
Description: description,
Unit: unit,
DatasetGroupID: groupID,
StartDate: startDate,
EndDate: endDate,
})
}
// ensureCategoryExists validates that categoryID references an existing