feat(datasets): periods (start_date/end_date), dg_id column, date filters; rename observation dates
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
@@ -71,20 +71,28 @@ type DatasetFilter struct {
|
||||
FileType *FileType
|
||||
Automated *bool
|
||||
Status *string
|
||||
// DatasetGroupID restricts the listing to one dataset group.
|
||||
DatasetGroupID *uuid.UUID
|
||||
// StartDateFrom / StartDateTo and EndDateFrom / EndDateTo are inclusive
|
||||
// YYYY-MM-DD bounds on the dataset's period columns.
|
||||
StartDateFrom *string
|
||||
StartDateTo *string
|
||||
EndDateFrom *string
|
||||
EndDateTo *string
|
||||
}
|
||||
|
||||
// Observation is a single unpivoted value from a dataset's attribute table,
|
||||
// keyed by KATO code and date (optionally a date..DateTo period). Exactly one
|
||||
// of Value / ValueText is typically set (numeric vs non-numeric cell); both
|
||||
// keyed by KATO code and date (optionally a StartDate..EndDate period). Exactly
|
||||
// one of Value / ValueText is typically set (numeric vs non-numeric cell); both
|
||||
// may be nil for an empty cell.
|
||||
type Observation struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
DatasetID uuid.UUID `json:"dataset_id"`
|
||||
KatoCode string `json:"kato_code"`
|
||||
Date string `json:"date"`
|
||||
// DateTo is the period's end date. Nil for a point-in-time observation
|
||||
StartDate string `json:"start_date"`
|
||||
// EndDate is the period's end date. Nil for a point-in-time observation
|
||||
// (pre-existing behavior).
|
||||
DateTo *string `json:"date_to,omitempty"`
|
||||
EndDate *string `json:"end_date,omitempty"`
|
||||
Value *float64 `json:"value"`
|
||||
ValueText *string `json:"value_text"`
|
||||
}
|
||||
@@ -185,21 +193,29 @@ type AttributeColumn struct {
|
||||
}
|
||||
|
||||
// YearColumn maps an attribute column to the date (or date period) it
|
||||
// represents, e.g. {"column": "F_2023", "date": "2023-01-01"} for a single
|
||||
// point in time, or {"column": "F_2020_2023", "date": "2020-01-01", "date_to":
|
||||
// "2023-12-31"} for a period. DateTo is optional; when nil the mapping is a
|
||||
// single point in time, preserving the original (pre-period) behavior.
|
||||
// represents, e.g. {"column": "F_2023", "start_date": "2023-01-01"} for a
|
||||
// single point in time, or {"column": "F_2020_2023", "start_date":
|
||||
// "2020-01-01", "end_date": "2023-12-31"} for a period. EndDate is optional;
|
||||
// when nil the mapping is a single point in time, preserving the original
|
||||
// (pre-period) behavior.
|
||||
type YearColumn struct {
|
||||
Column string `json:"column"`
|
||||
Date string `json:"date"`
|
||||
DateTo *string `json:"date_to,omitempty"`
|
||||
Column string `json:"column"`
|
||||
StartDate string `json:"start_date"`
|
||||
EndDate *string `json:"end_date,omitempty"`
|
||||
}
|
||||
|
||||
// DatasetSummary is the lightweight view of a dataset used in list responses.
|
||||
// It omits the heavy geometry/attribute/JSONB fields.
|
||||
type DatasetSummary struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
CategoryID uuid.UUID `json:"category_id"`
|
||||
ID uuid.UUID `json:"id"`
|
||||
CategoryID uuid.UUID `json:"category_id"`
|
||||
// DatasetGroupID links the dataset to its dataset group ("одноимённые"
|
||||
// datasets versioned by period). Nullable.
|
||||
DatasetGroupID *uuid.UUID `json:"dg_id"`
|
||||
// StartDate / EndDate is the period the dataset describes (YYYY-MM-DD).
|
||||
// Both nullable; EndDate drives the "freshest in group" rule.
|
||||
StartDate *string `json:"start_date"`
|
||||
EndDate *string `json:"end_date"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
Unit *string `json:"unit"`
|
||||
@@ -212,13 +228,20 @@ type DatasetSummary struct {
|
||||
|
||||
// Dataset is a geo file stored in the object store and grouped under a category.
|
||||
type Dataset struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
CategoryID uuid.UUID `json:"category_id"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
Unit *string `json:"unit"`
|
||||
Filename string `json:"filename"`
|
||||
StorageKey string `json:"storage_key"`
|
||||
ID uuid.UUID `json:"id"`
|
||||
CategoryID uuid.UUID `json:"category_id"`
|
||||
// DatasetGroupID links the dataset to its dataset group ("одноимённые"
|
||||
// datasets versioned by period). Nullable.
|
||||
DatasetGroupID *uuid.UUID `json:"dg_id"`
|
||||
// StartDate / EndDate is the period the dataset describes (YYYY-MM-DD).
|
||||
// Both nullable; EndDate drives the "freshest in group" rule.
|
||||
StartDate *string `json:"start_date"`
|
||||
EndDate *string `json:"end_date"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
Unit *string `json:"unit"`
|
||||
Filename string `json:"filename"`
|
||||
StorageKey string `json:"storage_key"`
|
||||
// CogStorageKey points to the Cloud-Optimized GeoTIFF for rasters. Nullable.
|
||||
CogStorageKey *string `json:"cog_storage_key"`
|
||||
FileType FileType `json:"file_type"`
|
||||
@@ -262,3 +285,37 @@ type Dataset struct {
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// DatasetFields is the set of editable metadata columns written by
|
||||
// UpdateFields. Nil pointers clear the corresponding nullable column.
|
||||
type DatasetFields struct {
|
||||
CategoryID uuid.UUID
|
||||
Name string
|
||||
Description *string
|
||||
Unit *string
|
||||
DatasetGroupID *uuid.UUID
|
||||
StartDate *string
|
||||
EndDate *string
|
||||
}
|
||||
|
||||
// ValidateDatePeriod checks that start/end (YYYY-MM-DD, either may be nil)
|
||||
// parse and that end is not before start. Returns an ErrValidation-wrapped
|
||||
// error naming the offending field.
|
||||
func ValidateDatePeriod(start, end *string) error {
|
||||
var from, to time.Time
|
||||
var err error
|
||||
if start != nil {
|
||||
if from, err = time.Parse("2006-01-02", *start); err != nil {
|
||||
return fmt.Errorf("%w: start_date %q is not YYYY-MM-DD", ErrValidation, *start)
|
||||
}
|
||||
}
|
||||
if end != nil {
|
||||
if to, err = time.Parse("2006-01-02", *end); err != nil {
|
||||
return fmt.Errorf("%w: end_date %q is not YYYY-MM-DD", ErrValidation, *end)
|
||||
}
|
||||
}
|
||||
if start != nil && end != nil && to.Before(from) {
|
||||
return fmt.Errorf("%w: end_date %q is before start_date %q", ErrValidation, *end, *start)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package domain
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFileTypeValid(t *testing.T) {
|
||||
valid := []FileType{FileTypeVectorWithKato, FileTypeVector, FileTypeRaster}
|
||||
@@ -72,3 +75,31 @@ func TestExtensionAllowedFor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDatePeriod(t *testing.T) {
|
||||
s := func(v string) *string { return &v }
|
||||
cases := []struct {
|
||||
name string
|
||||
start, end *string
|
||||
wantErr bool
|
||||
}{
|
||||
{"both nil", nil, nil, false},
|
||||
{"start only", s("2020-01-01"), nil, false},
|
||||
{"ordered", s("2020-01-01"), s("2020-12-31"), false},
|
||||
{"same day", s("2020-01-01"), s("2020-01-01"), false},
|
||||
{"reversed", s("2020-12-31"), s("2020-01-01"), true},
|
||||
{"bad start", s("01.01.2020"), nil, true},
|
||||
{"bad end", nil, s("2020-13-01"), true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := ValidateDatePeriod(tc.start, tc.end)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("err=%v wantErr=%v", err, tc.wantErr)
|
||||
}
|
||||
if err != nil && !errors.Is(err, ErrValidation) {
|
||||
t.Fatalf("expected ErrValidation, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ func NewDatasetRepository(pool *pgxpool.Pool) *DatasetRepository {
|
||||
// datasetColumns lists the dataset columns for SELECT and RETURNING. The
|
||||
// geometry is exposed as GeoJSON (jsonb) rather than its raw EWKB form, and a
|
||||
// bounding box array is derived for raster datasets only.
|
||||
const datasetColumns = `id, category_id, name, description, unit, filename, storage_key, cog_storage_key, file_type, size_bytes, content_type, properties, meta, automated, status, attribute_columns, kato_column, year_columns, parse_error, ST_AsGeoJSON(geometry)::jsonb AS geometry,
|
||||
const datasetColumns = `id, category_id, dg_id, to_char(start_date, 'YYYY-MM-DD') AS start_date, to_char(end_date, 'YYYY-MM-DD') AS end_date, name, description, unit, filename, storage_key, cog_storage_key, file_type, size_bytes, content_type, properties, meta, automated, status, attribute_columns, kato_column, year_columns, parse_error, ST_AsGeoJSON(geometry)::jsonb AS geometry,
|
||||
CASE WHEN file_type = 'raster' AND geometry IS NOT NULL
|
||||
THEN ARRAY[ST_XMin(geometry), ST_YMin(geometry), ST_XMax(geometry), ST_YMax(geometry)]
|
||||
ELSE NULL END AS bbox,
|
||||
@@ -37,7 +37,7 @@ const datasetColumns = `id, category_id, name, description, unit, filename, stor
|
||||
func scanDataset(row pgx.Row) (domain.Dataset, error) {
|
||||
var d domain.Dataset
|
||||
err := row.Scan(
|
||||
&d.ID, &d.CategoryID, &d.Name, &d.Description, &d.Unit,
|
||||
&d.ID, &d.CategoryID, &d.DatasetGroupID, &d.StartDate, &d.EndDate, &d.Name, &d.Description, &d.Unit,
|
||||
&d.Filename, &d.StorageKey, &d.CogStorageKey, &d.FileType, &d.SizeBytes, &d.ContentType,
|
||||
&d.Properties, &d.Meta, &d.Automated, &d.Status,
|
||||
&d.AttributeColumns, &d.KatoColumn, &d.YearColumns, &d.ParseError,
|
||||
@@ -59,28 +59,28 @@ func nullableJSON(raw json.RawMessage) any {
|
||||
// Create inserts a new dataset and returns the stored row.
|
||||
func (r *DatasetRepository) Create(ctx context.Context, d domain.Dataset) (domain.Dataset, error) {
|
||||
row := r.pool.QueryRow(ctx,
|
||||
`INSERT INTO datasets (category_id, name, description, unit, filename, storage_key, file_type, size_bytes, content_type, properties, meta, automated, status)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
`INSERT INTO datasets (category_id, name, description, unit, filename, storage_key, file_type, size_bytes, content_type, properties, meta, automated, status, dg_id, start_date, end_date)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15::date, $16::date)
|
||||
RETURNING `+datasetColumns,
|
||||
d.CategoryID, d.Name, d.Description, d.Unit, d.Filename, d.StorageKey, d.FileType, d.SizeBytes, d.ContentType,
|
||||
nullableJSON(d.Properties), nullableJSON(d.Meta), d.Automated, d.Status,
|
||||
d.DatasetGroupID, d.StartDate, d.EndDate,
|
||||
)
|
||||
out, err := scanDataset(row)
|
||||
return out, mapError(err)
|
||||
}
|
||||
|
||||
// UpdateFields overwrites a dataset's editable metadata columns (category,
|
||||
// name, description, unit). description and unit are nullable: a nil pointer
|
||||
// clears the column. Returns domain.ErrNotFound if the dataset does not
|
||||
// exist, or domain.ErrConflict if categoryID does not reference an existing
|
||||
// category.
|
||||
func (r *DatasetRepository) UpdateFields(ctx context.Context, id, categoryID uuid.UUID, name string, description, unit *string) (domain.Dataset, error) {
|
||||
// UpdateFields overwrites a dataset's editable metadata columns. Nil pointers
|
||||
// clear the nullable columns. Returns domain.ErrNotFound if the dataset does
|
||||
// not exist, or domain.ErrConflict if the category/group does not exist.
|
||||
func (r *DatasetRepository) UpdateFields(ctx context.Context, id uuid.UUID, f domain.DatasetFields) (domain.Dataset, error) {
|
||||
row := r.pool.QueryRow(ctx,
|
||||
`UPDATE datasets
|
||||
SET category_id = $2, name = $3, description = $4, unit = $5, updated_at = now()
|
||||
SET category_id = $2, name = $3, description = $4, unit = $5,
|
||||
dg_id = $6, start_date = $7::date, end_date = $8::date, updated_at = now()
|
||||
WHERE id = $1
|
||||
RETURNING `+datasetColumns,
|
||||
id, categoryID, name, description, unit,
|
||||
id, f.CategoryID, f.Name, f.Description, f.Unit, f.DatasetGroupID, f.StartDate, f.EndDate,
|
||||
)
|
||||
out, err := scanDataset(row)
|
||||
return out, mapError(err)
|
||||
@@ -276,23 +276,23 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u
|
||||
if len(obs) > 0 {
|
||||
rows := make([][]any, len(obs))
|
||||
for i, o := range obs {
|
||||
d, err := time.Parse("2006-01-02", o.Date)
|
||||
d, err := time.Parse("2006-01-02", o.StartDate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid observation date %q: %w", o.Date, err)
|
||||
return fmt.Errorf("invalid observation start_date %q: %w", o.StartDate, err)
|
||||
}
|
||||
var dateTo any
|
||||
if o.DateTo != nil {
|
||||
dt, err := time.Parse("2006-01-02", *o.DateTo)
|
||||
var endDate any
|
||||
if o.EndDate != nil {
|
||||
dt, err := time.Parse("2006-01-02", *o.EndDate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid observation date_to %q: %w", *o.DateTo, err)
|
||||
return fmt.Errorf("invalid observation end_date %q: %w", *o.EndDate, err)
|
||||
}
|
||||
dateTo = dt
|
||||
endDate = dt
|
||||
}
|
||||
rows[i] = []any{datasetID, o.KatoCode, d, dateTo, o.Value, o.ValueText}
|
||||
rows[i] = []any{datasetID, o.KatoCode, d, endDate, o.Value, o.ValueText}
|
||||
}
|
||||
_, err := tx.CopyFrom(ctx,
|
||||
pgx.Identifier{"dataset_observations"},
|
||||
[]string{"dataset_id", "kato_code", "date", "date_to", "value", "value_text"},
|
||||
[]string{"dataset_id", "kato_code", "start_date", "end_date", "value", "value_text"},
|
||||
pgx.CopyFromRows(rows),
|
||||
)
|
||||
if err != nil {
|
||||
@@ -303,16 +303,16 @@ func (r *DatasetRepository) ReplaceObservations(ctx context.Context, datasetID u
|
||||
return mapError(tx.Commit(ctx))
|
||||
}
|
||||
|
||||
const observationColumns = `id, dataset_id, kato_code, to_char(date, 'YYYY-MM-DD') AS date, to_char(date_to, 'YYYY-MM-DD') AS date_to, value, value_text`
|
||||
const observationColumns = `id, dataset_id, kato_code, to_char(start_date, 'YYYY-MM-DD') AS start_date, to_char(end_date, 'YYYY-MM-DD') AS end_date, value, value_text`
|
||||
|
||||
func scanObservation(row pgx.Row) (domain.Observation, error) {
|
||||
var o domain.Observation
|
||||
err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.Date, &o.DateTo, &o.Value, &o.ValueText)
|
||||
err := row.Scan(&o.ID, &o.DatasetID, &o.KatoCode, &o.StartDate, &o.EndDate, &o.Value, &o.ValueText)
|
||||
return o, err
|
||||
}
|
||||
|
||||
// ListObservations returns a page of observations for a dataset, optionally
|
||||
// filtered by KATO code, ordered by (kato_code, date).
|
||||
// filtered by KATO code, ordered by (kato_code, start_date).
|
||||
func (r *DatasetRepository) ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error) {
|
||||
base := `SELECT ` + observationColumns + ` FROM dataset_observations WHERE dataset_id = $1`
|
||||
|
||||
@@ -322,11 +322,11 @@ func (r *DatasetRepository) ListObservations(ctx context.Context, datasetID uuid
|
||||
)
|
||||
if katoCode != nil {
|
||||
rows, err = r.pool.Query(ctx,
|
||||
base+` AND kato_code = $2 ORDER BY kato_code, date LIMIT $3 OFFSET $4`,
|
||||
base+` AND kato_code = $2 ORDER BY kato_code, start_date LIMIT $3 OFFSET $4`,
|
||||
datasetID, *katoCode, limit, offset)
|
||||
} else {
|
||||
rows, err = r.pool.Query(ctx,
|
||||
base+` ORDER BY kato_code, date LIMIT $2 OFFSET $3`,
|
||||
base+` ORDER BY kato_code, start_date LIMIT $2 OFFSET $3`,
|
||||
datasetID, limit, offset)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -368,12 +368,25 @@ func (r *DatasetRepository) GetByID(ctx context.Context, id uuid.UUID) (domain.D
|
||||
return out, mapError(err)
|
||||
}
|
||||
|
||||
const datasetSummaryColumns = `id, category_id, name, description, unit, file_type, size_bytes, status, created_at, updated_at`
|
||||
// LatestInGroup returns the freshest dataset of a group: greatest end_date
|
||||
// (NULL end_date sorts last), ties broken by created_at DESC. status and
|
||||
// fileType optionally restrict candidates. Returns domain.ErrNotFound when no
|
||||
// dataset matches.
|
||||
func (r *DatasetRepository) LatestInGroup(ctx context.Context, groupID uuid.UUID, status *string, fileType *domain.FileType) (domain.Dataset, error) {
|
||||
where, args := datasetFilterClause(domain.DatasetFilter{DatasetGroupID: &groupID, Status: status, FileType: fileType}, nil)
|
||||
row := r.pool.QueryRow(ctx,
|
||||
`SELECT `+datasetColumns+` FROM datasets`+where+
|
||||
` ORDER BY end_date DESC NULLS LAST, created_at DESC LIMIT 1`, args...)
|
||||
out, err := scanDataset(row)
|
||||
return out, mapError(err)
|
||||
}
|
||||
|
||||
const datasetSummaryColumns = `id, category_id, dg_id, to_char(start_date, 'YYYY-MM-DD') AS start_date, to_char(end_date, 'YYYY-MM-DD') AS end_date, name, description, unit, file_type, size_bytes, status, created_at, updated_at`
|
||||
|
||||
func scanDatasetSummary(row pgx.Row) (domain.DatasetSummary, error) {
|
||||
var d domain.DatasetSummary
|
||||
err := row.Scan(
|
||||
&d.ID, &d.CategoryID, &d.Name, &d.Description, &d.Unit,
|
||||
&d.ID, &d.CategoryID, &d.DatasetGroupID, &d.StartDate, &d.EndDate, &d.Name, &d.Description, &d.Unit,
|
||||
&d.FileType, &d.SizeBytes, &d.Status, &d.CreatedAt, &d.UpdatedAt,
|
||||
)
|
||||
return d, err
|
||||
@@ -400,6 +413,22 @@ func datasetFilterClause(f domain.DatasetFilter, args []any) (string, []any) {
|
||||
args = append(args, *f.Status)
|
||||
conds = append(conds, fmt.Sprintf("status = $%d", len(args)))
|
||||
}
|
||||
if f.DatasetGroupID != nil {
|
||||
args = append(args, *f.DatasetGroupID)
|
||||
conds = append(conds, fmt.Sprintf("dg_id = $%d", len(args)))
|
||||
}
|
||||
for _, b := range []struct {
|
||||
col, op string
|
||||
val *string
|
||||
}{
|
||||
{"start_date", ">=", f.StartDateFrom}, {"start_date", "<=", f.StartDateTo},
|
||||
{"end_date", ">=", f.EndDateFrom}, {"end_date", "<=", f.EndDateTo},
|
||||
} {
|
||||
if b.val != nil {
|
||||
args = append(args, *b.val)
|
||||
conds = append(conds, fmt.Sprintf("%s %s $%d::date", b.col, b.op, len(args)))
|
||||
}
|
||||
}
|
||||
if len(conds) == 0 {
|
||||
return "", args
|
||||
}
|
||||
|
||||
@@ -9,14 +9,14 @@ import (
|
||||
)
|
||||
|
||||
// ListAllObservations returns every observation for a dataset ordered by
|
||||
// (kato_code, date). Unlike ListObservations it is unpaginated: it backs GeoJSON
|
||||
// (kato_code, start_date). Unlike ListObservations it is unpaginated: it backs GeoJSON
|
||||
// assembly, where all of a dataset's observations are needed at once.
|
||||
func (r *DatasetRepository) ListAllObservations(ctx context.Context, datasetID uuid.UUID) ([]domain.Observation, error) {
|
||||
rows, err := r.pool.Query(ctx,
|
||||
`SELECT `+observationColumns+`
|
||||
FROM dataset_observations
|
||||
WHERE dataset_id = $1
|
||||
ORDER BY kato_code, date`,
|
||||
ORDER BY kato_code, start_date`,
|
||||
datasetID)
|
||||
if err != nil {
|
||||
return nil, mapError(err)
|
||||
|
||||
@@ -37,7 +37,8 @@ var (
|
||||
type DatasetRepository interface {
|
||||
Create(ctx context.Context, d domain.Dataset) (domain.Dataset, error)
|
||||
GetByID(ctx context.Context, id uuid.UUID) (domain.Dataset, error)
|
||||
UpdateFields(ctx context.Context, id, categoryID uuid.UUID, name string, description, unit *string) (domain.Dataset, error)
|
||||
UpdateFields(ctx context.Context, id uuid.UUID, f domain.DatasetFields) (domain.Dataset, error)
|
||||
LatestInGroup(ctx context.Context, groupID uuid.UUID, status *string, fileType *domain.FileType) (domain.Dataset, error)
|
||||
ListSummaries(ctx context.Context, filter domain.DatasetFilter, limit, offset int) ([]domain.DatasetSummary, error)
|
||||
Count(ctx context.Context, filter domain.DatasetFilter) (int, error)
|
||||
Delete(ctx context.Context, id uuid.UUID) error
|
||||
@@ -125,6 +126,11 @@ type UploadInput struct {
|
||||
ContentType string
|
||||
Size int64
|
||||
Reader io.Reader
|
||||
// DatasetGroupID optionally files the dataset under a dataset group.
|
||||
DatasetGroupID *uuid.UUID
|
||||
// StartDate / EndDate is the period the dataset describes (YYYY-MM-DD).
|
||||
StartDate *string
|
||||
EndDate *string
|
||||
}
|
||||
|
||||
// DatasetService implements dataset business rules and object storage handling.
|
||||
@@ -190,6 +196,9 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat
|
||||
if err := s.ensureCategoryExists(ctx, in.CategoryID); err != nil {
|
||||
return domain.Dataset{}, err
|
||||
}
|
||||
if err := domain.ValidateDatePeriod(in.StartDate, in.EndDate); err != nil {
|
||||
return domain.Dataset{}, err
|
||||
}
|
||||
|
||||
name := in.Name
|
||||
if name == "" {
|
||||
@@ -210,18 +219,21 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat
|
||||
}
|
||||
|
||||
dataset, err := s.repo.Create(ctx, domain.Dataset{
|
||||
CategoryID: in.CategoryID,
|
||||
Name: name,
|
||||
Description: in.Description,
|
||||
Unit: in.Unit,
|
||||
Meta: in.Meta,
|
||||
Automated: in.Automated,
|
||||
Status: status,
|
||||
Filename: in.Filename,
|
||||
StorageKey: storageKey,
|
||||
FileType: in.FileType,
|
||||
SizeBytes: in.Size,
|
||||
ContentType: in.ContentType,
|
||||
CategoryID: in.CategoryID,
|
||||
DatasetGroupID: in.DatasetGroupID,
|
||||
StartDate: in.StartDate,
|
||||
EndDate: in.EndDate,
|
||||
Name: name,
|
||||
Description: in.Description,
|
||||
Unit: in.Unit,
|
||||
Meta: in.Meta,
|
||||
Automated: in.Automated,
|
||||
Status: status,
|
||||
Filename: in.Filename,
|
||||
StorageKey: storageKey,
|
||||
FileType: in.FileType,
|
||||
SizeBytes: in.Size,
|
||||
ContentType: in.ContentType,
|
||||
})
|
||||
if err != nil {
|
||||
// Compensate: the row was not written, so the object would be orphaned.
|
||||
@@ -682,17 +694,17 @@ func (s *DatasetService) SaveMapping(ctx context.Context, id uuid.UUID, in Mappi
|
||||
if _, ok := known[yc.Column]; !ok {
|
||||
return domain.Dataset{}, fmt.Errorf("%w: year column %q is not among the detected columns", domain.ErrValidation, yc.Column)
|
||||
}
|
||||
from, err := time.Parse("2006-01-02", yc.Date)
|
||||
from, err := time.Parse("2006-01-02", yc.StartDate)
|
||||
if err != nil {
|
||||
return domain.Dataset{}, fmt.Errorf("%w: invalid date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.Date, yc.Column)
|
||||
return domain.Dataset{}, fmt.Errorf("%w: invalid start_date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, yc.StartDate, yc.Column)
|
||||
}
|
||||
if yc.DateTo != nil {
|
||||
to, err := time.Parse("2006-01-02", *yc.DateTo)
|
||||
if yc.EndDate != nil {
|
||||
to, err := time.Parse("2006-01-02", *yc.EndDate)
|
||||
if err != nil {
|
||||
return domain.Dataset{}, fmt.Errorf("%w: invalid date_to %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, *yc.DateTo, yc.Column)
|
||||
return domain.Dataset{}, fmt.Errorf("%w: invalid end_date %q for column %q (want YYYY-MM-DD)", domain.ErrValidation, *yc.EndDate, yc.Column)
|
||||
}
|
||||
if to.Before(from) {
|
||||
return domain.Dataset{}, fmt.Errorf("%w: date_to %q is before date %q for column %q", domain.ErrValidation, *yc.DateTo, yc.Date, yc.Column)
|
||||
return domain.Dataset{}, fmt.Errorf("%w: end_date %q is before start_date %q for column %q", domain.ErrValidation, *yc.EndDate, yc.StartDate, yc.Column)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -748,9 +760,10 @@ func (s *DatasetService) Extract(ctx context.Context, id uuid.UUID) error {
|
||||
}
|
||||
|
||||
// buildObservations unpivots rows into observations. Rows without a KATO code
|
||||
// are skipped; duplicate (kato, date, date_to) tuples keep the last value.
|
||||
// Numeric cells populate Value, others ValueText. A year column's DateTo (nil
|
||||
// for a point-in-time mapping) carries through to the observation unchanged.
|
||||
// are skipped; duplicate (kato, start_date, end_date) tuples keep the last
|
||||
// value. Numeric cells populate Value, others ValueText. A year column's
|
||||
// EndDate (nil for a point-in-time mapping) carries through to the observation
|
||||
// unchanged.
|
||||
func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.YearColumn, rows []map[string]string) []domain.Observation {
|
||||
obs := make([]domain.Observation, 0, len(rows)*len(years))
|
||||
index := make(map[string]int)
|
||||
@@ -761,7 +774,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
|
||||
continue
|
||||
}
|
||||
for _, yc := range years {
|
||||
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, Date: yc.Date, DateTo: yc.DateTo}
|
||||
o := domain.Observation{DatasetID: datasetID, KatoCode: kato, StartDate: yc.StartDate, EndDate: yc.EndDate}
|
||||
if raw := strings.TrimSpace(row[yc.Column]); raw != "" {
|
||||
if f, err := strconv.ParseFloat(raw, 64); err == nil {
|
||||
o.Value = &f
|
||||
@@ -769,7 +782,7 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
|
||||
o.ValueText = &raw
|
||||
}
|
||||
}
|
||||
key := kato + "\x00" + yc.Date + "\x00" + dateToKey(yc.DateTo)
|
||||
key := kato + "\x00" + yc.StartDate + "\x00" + endDateKey(yc.EndDate)
|
||||
if i, ok := index[key]; ok {
|
||||
obs[i] = o
|
||||
} else {
|
||||
@@ -781,12 +794,12 @@ func buildObservations(datasetID uuid.UUID, katoColumn string, years []domain.Ye
|
||||
return obs
|
||||
}
|
||||
|
||||
// dateToKey normalizes an optional period end date for use in a dedup key.
|
||||
func dateToKey(dateTo *string) string {
|
||||
if dateTo == nil {
|
||||
// endDateKey normalizes an optional period end date for use in a dedup key.
|
||||
func endDateKey(endDate *string) string {
|
||||
if endDate == nil {
|
||||
return ""
|
||||
}
|
||||
return *dateTo
|
||||
return *endDate
|
||||
}
|
||||
|
||||
// ObservationPage is a page of observations with pagination metadata.
|
||||
@@ -888,7 +901,7 @@ func (s *DatasetService) generateGeoJSON(ctx context.Context, id uuid.UUID) erro
|
||||
// boundary taken from the districts table and its observation values nested
|
||||
// under a `data` object (keyed by date) alongside `kato` and `name`. KATO codes
|
||||
// with no matching district are skipped. When any observation carries a period
|
||||
// (DateTo set), its end date is additionally exposed under a `periods` object
|
||||
// (EndDate set), its end date is additionally exposed under a `periods` object
|
||||
// (keyed by the same start date used in `data`); `periods` is omitted entirely
|
||||
// when every observation is point-in-time, so a dataset mapped without periods
|
||||
// produces byte-identical output to before periods existed.
|
||||
@@ -954,7 +967,7 @@ func (s *DatasetService) loadGeoJSONDataset(ctx context.Context, id uuid.UUID) (
|
||||
// the grouped observation values (keyed by date) nested under a `data` object
|
||||
// alongside `kato` and `name` in each Feature's properties. When periods holds
|
||||
// an entry for the KATO (at least one observation had a period), its date->
|
||||
// date_to map is additionally nested under a `periods` object; a KATO with no
|
||||
// end_date map is additionally nested under a `periods` object; a KATO with no
|
||||
// period observations gets no `periods` key at all. KATO codes with no
|
||||
// matching district are skipped. order drives the deterministic feature order.
|
||||
func (s *DatasetService) districtFeatures(ctx context.Context, grouped map[string]map[string]any, periods map[string]map[string]string, order []string) ([]domain.Feature, error) {
|
||||
@@ -1018,8 +1031,8 @@ func vectorFeatureProperties(raw json.RawMessage) map[string]any {
|
||||
|
||||
// groupObservationsByKato groups observations by KATO code into date->value
|
||||
// maps. The value is the numeric Value when present, otherwise ValueText,
|
||||
// otherwise nil (an empty cell). It also groups, per KATO, a date->date_to map
|
||||
// for observations that carry a period (DateTo set); a KATO with no period
|
||||
// otherwise nil (an empty cell). It also groups, per KATO, a start_date->end_date map
|
||||
// for observations that carry a period (EndDate set); a KATO with no period
|
||||
// observations gets no entry in periods at all, so callers can tell "no
|
||||
// periods" apart from "empty periods". order lists KATO codes in first-seen
|
||||
// order so the resulting feature order is deterministic.
|
||||
@@ -1034,15 +1047,15 @@ func groupObservationsByKato(obs []domain.Observation) (grouped map[string]map[s
|
||||
grouped[o.KatoCode] = values
|
||||
order = append(order, o.KatoCode)
|
||||
}
|
||||
values[o.Date] = observationValue(o)
|
||||
values[o.StartDate] = observationValue(o)
|
||||
|
||||
if o.DateTo != nil {
|
||||
if o.EndDate != nil {
|
||||
p, ok := periods[o.KatoCode]
|
||||
if !ok {
|
||||
p = make(map[string]string)
|
||||
periods[o.KatoCode] = p
|
||||
}
|
||||
p[o.Date] = *o.DateTo
|
||||
p[o.StartDate] = *o.EndDate
|
||||
}
|
||||
}
|
||||
return grouped, periods, order
|
||||
@@ -1074,6 +1087,10 @@ type DatasetUpdateInput struct {
|
||||
Name string
|
||||
Description *string
|
||||
Unit *string
|
||||
// DatasetGroupID, StartDate and EndDate are nullable: nil clears them.
|
||||
DatasetGroupID *uuid.UUID
|
||||
StartDate *string
|
||||
EndDate *string
|
||||
}
|
||||
|
||||
// Update replaces a dataset's editable metadata (category, name, description,
|
||||
@@ -1082,7 +1099,18 @@ func (s *DatasetService) Update(ctx context.Context, id uuid.UUID, in DatasetUpd
|
||||
if err := s.ensureCategoryExists(ctx, in.CategoryID); err != nil {
|
||||
return domain.Dataset{}, err
|
||||
}
|
||||
return s.repo.UpdateFields(ctx, id, in.CategoryID, in.Name, in.Description, in.Unit)
|
||||
if err := domain.ValidateDatePeriod(in.StartDate, in.EndDate); err != nil {
|
||||
return domain.Dataset{}, err
|
||||
}
|
||||
return s.repo.UpdateFields(ctx, id, domain.DatasetFields{
|
||||
CategoryID: in.CategoryID,
|
||||
Name: in.Name,
|
||||
Description: in.Description,
|
||||
Unit: in.Unit,
|
||||
DatasetGroupID: in.DatasetGroupID,
|
||||
StartDate: in.StartDate,
|
||||
EndDate: in.EndDate,
|
||||
})
|
||||
}
|
||||
|
||||
// DatasetPatchInput carries a sparse set of metadata field changes. A nil
|
||||
@@ -1096,6 +1124,13 @@ type DatasetPatchInput struct {
|
||||
DescriptionSet bool
|
||||
Unit *string
|
||||
UnitSet bool
|
||||
|
||||
DatasetGroupID *uuid.UUID
|
||||
DatasetGroupIDSet bool
|
||||
StartDate *string
|
||||
StartDateSet bool
|
||||
EndDate *string
|
||||
EndDateSet bool
|
||||
}
|
||||
|
||||
// Patch applies a sparse update to a dataset's editable metadata (category,
|
||||
@@ -1125,8 +1160,31 @@ func (s *DatasetService) Patch(ctx context.Context, id uuid.UUID, in DatasetPatc
|
||||
if in.UnitSet {
|
||||
unit = in.Unit
|
||||
}
|
||||
groupID := current.DatasetGroupID
|
||||
if in.DatasetGroupIDSet {
|
||||
groupID = in.DatasetGroupID
|
||||
}
|
||||
startDate := current.StartDate
|
||||
if in.StartDateSet {
|
||||
startDate = in.StartDate
|
||||
}
|
||||
endDate := current.EndDate
|
||||
if in.EndDateSet {
|
||||
endDate = in.EndDate
|
||||
}
|
||||
if err := domain.ValidateDatePeriod(startDate, endDate); err != nil {
|
||||
return domain.Dataset{}, err
|
||||
}
|
||||
|
||||
return s.repo.UpdateFields(ctx, id, categoryID, name, description, unit)
|
||||
return s.repo.UpdateFields(ctx, id, domain.DatasetFields{
|
||||
CategoryID: categoryID,
|
||||
Name: name,
|
||||
Description: description,
|
||||
Unit: unit,
|
||||
DatasetGroupID: groupID,
|
||||
StartDate: startDate,
|
||||
EndDate: endDate,
|
||||
})
|
||||
}
|
||||
|
||||
// ensureCategoryExists validates that categoryID references an existing
|
||||
|
||||
@@ -56,19 +56,60 @@ func (r *stubDatasetRepo) GetByID(_ context.Context, id uuid.UUID) (domain.Datas
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (r *stubDatasetRepo) UpdateFields(_ context.Context, id, categoryID uuid.UUID, name string, description, unit *string) (domain.Dataset, error) {
|
||||
func (r *stubDatasetRepo) UpdateFields(_ context.Context, id uuid.UUID, f domain.DatasetFields) (domain.Dataset, error) {
|
||||
d, ok := r.store[id]
|
||||
if !ok {
|
||||
return domain.Dataset{}, domain.ErrNotFound
|
||||
}
|
||||
d.CategoryID = categoryID
|
||||
d.Name = name
|
||||
d.Description = description
|
||||
d.Unit = unit
|
||||
d.CategoryID = f.CategoryID
|
||||
d.Name = f.Name
|
||||
d.Description = f.Description
|
||||
d.Unit = f.Unit
|
||||
d.DatasetGroupID = f.DatasetGroupID
|
||||
d.StartDate = f.StartDate
|
||||
d.EndDate = f.EndDate
|
||||
r.store[id] = d
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (r *stubDatasetRepo) LatestInGroup(_ context.Context, groupID uuid.UUID, status *string, fileType *domain.FileType) (domain.Dataset, error) {
|
||||
var best *domain.Dataset
|
||||
for _, d := range r.store {
|
||||
d := d
|
||||
if d.DatasetGroupID == nil || *d.DatasetGroupID != groupID {
|
||||
continue
|
||||
}
|
||||
if status != nil && d.Status != *status {
|
||||
continue
|
||||
}
|
||||
if fileType != nil && d.FileType != *fileType {
|
||||
continue
|
||||
}
|
||||
if best == nil || laterPeriod(d, *best) {
|
||||
best = &d
|
||||
}
|
||||
}
|
||||
if best == nil {
|
||||
return domain.Dataset{}, domain.ErrNotFound
|
||||
}
|
||||
return *best, nil
|
||||
}
|
||||
|
||||
// laterPeriod reports whether a is fresher than b: greater end_date, NULL
|
||||
// end_date last, then newer created_at.
|
||||
func laterPeriod(a, b domain.Dataset) bool {
|
||||
switch {
|
||||
case a.EndDate != nil && b.EndDate == nil:
|
||||
return true
|
||||
case a.EndDate == nil && b.EndDate != nil:
|
||||
return false
|
||||
case a.EndDate != nil && b.EndDate != nil && *a.EndDate != *b.EndDate:
|
||||
return *a.EndDate > *b.EndDate
|
||||
default:
|
||||
return a.CreatedAt.After(b.CreatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *stubDatasetRepo) ListSummaries(_ context.Context, filter domain.DatasetFilter, limit, offset int) ([]domain.DatasetSummary, error) {
|
||||
r.lastFilter = filter
|
||||
r.lastLimit = limit
|
||||
@@ -756,7 +797,7 @@ func TestDatasetService_SaveMapping(t *testing.T) {
|
||||
Status: domain.DatasetStatusAwaitingMapping,
|
||||
AttributeColumns: []domain.AttributeColumn{{Name: "като"}, {Name: "F_2023"}},
|
||||
}
|
||||
validYears := []domain.YearColumn{{Column: "F_2023", Date: "2023-01-01"}}
|
||||
validYears := []domain.YearColumn{{Column: "F_2023", StartDate: "2023-01-01"}}
|
||||
|
||||
newSvc := func() (*stubDatasetRepo, *DatasetService) {
|
||||
repo := newStubDatasetRepo()
|
||||
@@ -774,31 +815,31 @@ func TestDatasetService_SaveMapping(t *testing.T) {
|
||||
|
||||
t.Run("unknown year column", func(t *testing.T) {
|
||||
_, svc := newSvc()
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "X", Date: "2023-01-01"}}})
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "X", StartDate: "2023-01-01"}}})
|
||||
if !errors.Is(err, domain.ErrValidation) {
|
||||
t.Fatalf("want ErrValidation, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("bad date", func(t *testing.T) {
|
||||
t.Run("bad start_date", func(t *testing.T) {
|
||||
_, svc := newSvc()
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023"}}})
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", StartDate: "2023"}}})
|
||||
if !errors.Is(err, domain.ErrValidation) {
|
||||
t.Fatalf("want ErrValidation, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("bad date_to", func(t *testing.T) {
|
||||
t.Run("bad end_date", func(t *testing.T) {
|
||||
_, svc := newSvc()
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-01-01", DateTo: strPtr("2023")}}})
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", StartDate: "2023-01-01", EndDate: strPtr("2023")}}})
|
||||
if !errors.Is(err, domain.ErrValidation) {
|
||||
t.Fatalf("want ErrValidation, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("date_to before date", func(t *testing.T) {
|
||||
t.Run("end_date before start_date", func(t *testing.T) {
|
||||
_, svc := newSvc()
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-12-31", DateTo: strPtr("2023-01-01")}}})
|
||||
_, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: []domain.YearColumn{{Column: "F_2023", StartDate: "2023-12-31", EndDate: strPtr("2023-01-01")}}})
|
||||
if !errors.Is(err, domain.ErrValidation) {
|
||||
t.Fatalf("want ErrValidation, got %v", err)
|
||||
}
|
||||
@@ -808,12 +849,12 @@ func TestDatasetService_SaveMapping(t *testing.T) {
|
||||
repo, svc := newSvc()
|
||||
got, err := svc.SaveMapping(ctx, id, MappingInput{
|
||||
KatoColumn: "като",
|
||||
YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2020-01-01", DateTo: strPtr("2023-12-31")}},
|
||||
YearColumns: []domain.YearColumn{{Column: "F_2023", StartDate: "2020-01-01", EndDate: strPtr("2023-12-31")}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(got.YearColumns) != 1 || got.YearColumns[0].DateTo == nil || *got.YearColumns[0].DateTo != "2023-12-31" {
|
||||
if len(got.YearColumns) != 1 || got.YearColumns[0].EndDate == nil || *got.YearColumns[0].EndDate != "2023-12-31" {
|
||||
t.Fatalf("period not saved: %+v", got.YearColumns)
|
||||
}
|
||||
if len(repo.store[id].YearColumns) != 1 {
|
||||
@@ -866,8 +907,8 @@ func TestDatasetService_SaveMapping(t *testing.T) {
|
||||
func TestBuildObservations(t *testing.T) {
|
||||
id := uuid.New()
|
||||
years := []domain.YearColumn{
|
||||
{Column: "F_2023", Date: "2023-01-01"},
|
||||
{Column: "D_2025", Date: "2025-01-01"},
|
||||
{Column: "F_2023", StartDate: "2023-01-01"},
|
||||
{Column: "D_2025", StartDate: "2025-01-01"},
|
||||
}
|
||||
rows := []map[string]string{
|
||||
{"като": "751010000", "F_2023": "100", "D_2025": "n/a"},
|
||||
@@ -882,7 +923,7 @@ func TestBuildObservations(t *testing.T) {
|
||||
|
||||
byKey := map[string]domain.Observation{}
|
||||
for _, o := range obs {
|
||||
byKey[o.KatoCode+"|"+o.Date] = o
|
||||
byKey[o.KatoCode+"|"+o.StartDate] = o
|
||||
}
|
||||
if o := byKey["751010000|2023-01-01"]; o.Value == nil || *o.Value != 100 {
|
||||
t.Errorf("numeric cell not stored as value: %+v", o)
|
||||
@@ -891,8 +932,8 @@ func TestBuildObservations(t *testing.T) {
|
||||
t.Errorf("non-numeric cell not stored as value_text: %+v", o)
|
||||
}
|
||||
for _, o := range obs {
|
||||
if o.DateTo != nil {
|
||||
t.Fatalf("point-in-time mapping must not set DateTo, got %+v", o)
|
||||
if o.EndDate != nil {
|
||||
t.Fatalf("point-in-time mapping must not set EndDate, got %+v", o)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -900,8 +941,8 @@ func TestBuildObservations(t *testing.T) {
|
||||
func TestBuildObservations_Period(t *testing.T) {
|
||||
id := uuid.New()
|
||||
years := []domain.YearColumn{
|
||||
{Column: "F_2020_2023", Date: "2020-01-01", DateTo: strPtr("2023-12-31")}, // period
|
||||
{Column: "F_2024", Date: "2024-01-01"}, // point-in-time, unaffected
|
||||
{Column: "F_2020_2023", StartDate: "2020-01-01", EndDate: strPtr("2023-12-31")}, // period
|
||||
{Column: "F_2024", StartDate: "2024-01-01"}, // point-in-time, unaffected
|
||||
}
|
||||
rows := []map[string]string{
|
||||
{"като": "751010000", "F_2020_2023": "100", "F_2024": "200"},
|
||||
@@ -912,22 +953,22 @@ func TestBuildObservations_Period(t *testing.T) {
|
||||
t.Fatalf("want 2 observations, got %d", len(obs))
|
||||
}
|
||||
|
||||
byDate := map[string]domain.Observation{}
|
||||
byStartDate := map[string]domain.Observation{}
|
||||
for _, o := range obs {
|
||||
byDate[o.Date] = o
|
||||
byStartDate[o.StartDate] = o
|
||||
}
|
||||
|
||||
period := byDate["2020-01-01"]
|
||||
if period.DateTo == nil || *period.DateTo != "2023-12-31" {
|
||||
t.Fatalf("period observation missing date_to: %+v", period)
|
||||
period := byStartDate["2020-01-01"]
|
||||
if period.EndDate == nil || *period.EndDate != "2023-12-31" {
|
||||
t.Fatalf("period observation missing end_date: %+v", period)
|
||||
}
|
||||
if period.Value == nil || *period.Value != 100 {
|
||||
t.Fatalf("period observation value wrong: %+v", period)
|
||||
}
|
||||
|
||||
point := byDate["2024-01-01"]
|
||||
if point.DateTo != nil {
|
||||
t.Fatalf("point-in-time observation must have nil date_to (backward compat), got %+v", point)
|
||||
point := byStartDate["2024-01-01"]
|
||||
if point.EndDate != nil {
|
||||
t.Fatalf("point-in-time observation must have nil end_date (backward compat), got %+v", point)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -940,7 +981,7 @@ func TestDatasetService_Extract(t *testing.T) {
|
||||
Filename: "r.geojson", StorageKey: "k",
|
||||
Status: domain.DatasetStatusExtracting,
|
||||
KatoColumn: &kato,
|
||||
YearColumns: []domain.YearColumn{{Column: "F_2023", Date: "2023-01-01"}},
|
||||
YearColumns: []domain.YearColumn{{Column: "F_2023", StartDate: "2023-01-01"}},
|
||||
}
|
||||
repo.districts["751010000"] = domain.District{
|
||||
Kato: "751010000", Name: "Almaty",
|
||||
@@ -1002,9 +1043,9 @@ func TestDatasetService_GeoJSON_DistrictJoin(t *testing.T) {
|
||||
|
||||
v2020, v2021 := 100.0, 150.0
|
||||
repo.observations[id] = []domain.Observation{
|
||||
{KatoCode: "710000000", Date: "2020-01-01", Value: &v2020},
|
||||
{KatoCode: "710000000", Date: "2021-01-01", Value: &v2021},
|
||||
{KatoCode: "999999999", Date: "2020-01-01", Value: &v2020}, // no district -> skipped
|
||||
{KatoCode: "710000000", StartDate: "2020-01-01", Value: &v2020},
|
||||
{KatoCode: "710000000", StartDate: "2021-01-01", Value: &v2021},
|
||||
{KatoCode: "999999999", StartDate: "2020-01-01", Value: &v2020}, // no district -> skipped
|
||||
}
|
||||
repo.districts["710000000"] = domain.District{
|
||||
Kato: "710000000", Name: "Astana",
|
||||
@@ -1050,8 +1091,8 @@ func TestDatasetService_GeoJSON_Period(t *testing.T) {
|
||||
|
||||
period, point := 100.0, 200.0
|
||||
repo.observations[id] = []domain.Observation{
|
||||
{KatoCode: "710000000", Date: "2020-01-01", DateTo: strPtr("2023-12-31"), Value: &period},
|
||||
{KatoCode: "710000000", Date: "2024-01-01", Value: &point}, // point-in-time, no period
|
||||
{KatoCode: "710000000", StartDate: "2020-01-01", EndDate: strPtr("2023-12-31"), Value: &period},
|
||||
{KatoCode: "710000000", StartDate: "2024-01-01", Value: &point}, // point-in-time, no period
|
||||
}
|
||||
repo.districts["710000000"] = domain.District{
|
||||
Kato: "710000000", Name: "Astana",
|
||||
@@ -1092,7 +1133,7 @@ func TestDatasetService_GeoJSON_NoPeriods_OmitsPeriodsKey(t *testing.T) {
|
||||
repo := newStubDatasetRepo()
|
||||
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeVectorWithKato, Status: domain.DatasetStatusReady}
|
||||
v := 100.0
|
||||
repo.observations[id] = []domain.Observation{{KatoCode: "710000000", Date: "2020-01-01", Value: &v}}
|
||||
repo.observations[id] = []domain.Observation{{KatoCode: "710000000", StartDate: "2020-01-01", Value: &v}}
|
||||
repo.districts["710000000"] = domain.District{
|
||||
Kato: "710000000", Name: "Astana",
|
||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`),
|
||||
@@ -1116,8 +1157,8 @@ func TestDatasetService_GeoJSON_IgnoresDatasetGeometry(t *testing.T) {
|
||||
}
|
||||
v := 100.0
|
||||
repo.observations[id] = []domain.Observation{
|
||||
{KatoCode: "710000000", Date: "2020-01-01", Value: &v},
|
||||
{KatoCode: "999999999", Date: "2020-01-01", Value: &v}, // no district -> skipped
|
||||
{KatoCode: "710000000", StartDate: "2020-01-01", Value: &v},
|
||||
{KatoCode: "999999999", StartDate: "2020-01-01", Value: &v}, // no district -> skipped
|
||||
}
|
||||
district := json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`)
|
||||
repo.districts["710000000"] = domain.District{Kato: "710000000", Name: "Astana", Geometry: district}
|
||||
@@ -1174,7 +1215,7 @@ func TestDatasetService_GeoJSON_LazilyGeneratesAndCaches(t *testing.T) {
|
||||
repo := newStubDatasetRepo()
|
||||
repo.store[id] = domain.Dataset{ID: id, FileType: domain.FileTypeVectorWithKato, Status: domain.DatasetStatusReady}
|
||||
v := 100.0
|
||||
repo.observations[id] = []domain.Observation{{KatoCode: "710000000", Date: "2020-01-01", Value: &v}}
|
||||
repo.observations[id] = []domain.Observation{{KatoCode: "710000000", StartDate: "2020-01-01", Value: &v}}
|
||||
repo.districts["710000000"] = domain.District{
|
||||
Kato: "710000000", Name: "Astana",
|
||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[71,51],[72,51],[72,52],[71,51]]]}`),
|
||||
@@ -1694,3 +1735,40 @@ func TestDatasetService_Patch_NotFound(t *testing.T) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_Upload_StoresPeriodAndGroup(t *testing.T) {
|
||||
repo := newStubDatasetRepo()
|
||||
svc := newDatasetService(repo, &stubStore{}, true)
|
||||
|
||||
gid := uuid.New()
|
||||
start, end := "2020-01-01", "2020-12-31"
|
||||
in := validUpload()
|
||||
in.DatasetGroupID = &gid
|
||||
in.StartDate = &start
|
||||
in.EndDate = &end
|
||||
|
||||
ds, err := svc.Upload(context.Background(), in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ds.DatasetGroupID == nil || *ds.DatasetGroupID != gid || *ds.StartDate != start || *ds.EndDate != end {
|
||||
t.Fatalf("period/group not stored: %+v", ds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_Upload_RejectsReversedPeriod(t *testing.T) {
|
||||
store := &stubStore{}
|
||||
svc := newDatasetService(newStubDatasetRepo(), store, true)
|
||||
|
||||
start, end := "2021-01-01", "2020-12-31"
|
||||
in := validUpload()
|
||||
in.StartDate = &start
|
||||
in.EndDate = &end
|
||||
|
||||
if _, err := svc.Upload(context.Background(), in); !errors.Is(err, domain.ErrValidation) {
|
||||
t.Fatalf("want ErrValidation, got %v", err)
|
||||
}
|
||||
if len(store.put) != 0 {
|
||||
t.Fatalf("nothing should be uploaded on validation failure, got %v", store.put)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,9 +80,9 @@ func (h *DatasetHandler) status(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
type yearColumnInput struct {
|
||||
Column string `json:"column" validate:"required"`
|
||||
Date string `json:"date" validate:"required,datetime=2006-01-02"`
|
||||
DateTo *string `json:"date_to" validate:"omitempty,datetime=2006-01-02"`
|
||||
Column string `json:"column" validate:"required"`
|
||||
StartDate string `json:"start_date" validate:"required,datetime=2006-01-02"`
|
||||
EndDate *string `json:"end_date" validate:"omitempty,datetime=2006-01-02"`
|
||||
}
|
||||
|
||||
type mappingRequest struct {
|
||||
@@ -109,7 +109,7 @@ func (h *DatasetHandler) mapping(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
in := service.MappingInput{KatoColumn: req.KatoColumn}
|
||||
for _, yc := range req.YearColumns {
|
||||
in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, Date: yc.Date, DateTo: yc.DateTo})
|
||||
in.YearColumns = append(in.YearColumns, domain.YearColumn{Column: yc.Column, StartDate: yc.StartDate, EndDate: yc.EndDate})
|
||||
}
|
||||
|
||||
dataset, err := h.svc.SaveMapping(r.Context(), id, in)
|
||||
@@ -150,6 +150,17 @@ func (h *DatasetHandler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// The dataset group is optional; the period is validated by the service.
|
||||
var groupID *uuid.UUID
|
||||
if v := optionalFormValue(r, "dg_id"); v != nil {
|
||||
gid, err := uuid.Parse(*v)
|
||||
if err != nil {
|
||||
httputil.WriteError(w, http.StatusUnprocessableEntity, "dg_id must be a valid UUID")
|
||||
return
|
||||
}
|
||||
groupID = &gid
|
||||
}
|
||||
|
||||
file, header, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
httputil.WriteError(w, http.StatusBadRequest, "file is required")
|
||||
@@ -163,17 +174,20 @@ func (h *DatasetHandler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
dataset, err := h.svc.Upload(r.Context(), service.UploadInput{
|
||||
CategoryID: categoryID,
|
||||
Name: r.FormValue("name"),
|
||||
Description: optionalFormValue(r, "description"),
|
||||
Unit: optionalFormValue(r, "unit"),
|
||||
Meta: meta,
|
||||
Automated: automated,
|
||||
Filename: header.Filename,
|
||||
FileType: fileType,
|
||||
ContentType: contentType,
|
||||
Size: header.Size,
|
||||
Reader: file,
|
||||
CategoryID: categoryID,
|
||||
Name: r.FormValue("name"),
|
||||
Description: optionalFormValue(r, "description"),
|
||||
Unit: optionalFormValue(r, "unit"),
|
||||
Meta: meta,
|
||||
Automated: automated,
|
||||
Filename: header.Filename,
|
||||
FileType: fileType,
|
||||
ContentType: contentType,
|
||||
Size: header.Size,
|
||||
Reader: file,
|
||||
DatasetGroupID: groupID,
|
||||
StartDate: optionalFormValue(r, "start_date"),
|
||||
EndDate: optionalFormValue(r, "end_date"),
|
||||
})
|
||||
if err != nil {
|
||||
respondDomainError(w, err)
|
||||
@@ -226,7 +240,32 @@ func (h *DatasetHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
categoryCode = &v
|
||||
}
|
||||
|
||||
filter := domain.DatasetFilter{CategoryID: categoryID}
|
||||
groupID, ok := parseOptionalUUIDQuery(w, r, "dg_id")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
filter := domain.DatasetFilter{CategoryID: categoryID, DatasetGroupID: groupID}
|
||||
|
||||
for _, b := range []struct {
|
||||
name string
|
||||
dst **string
|
||||
}{
|
||||
{"start_date_from", &filter.StartDateFrom},
|
||||
{"start_date_to", &filter.StartDateTo},
|
||||
{"end_date_from", &filter.EndDateFrom},
|
||||
{"end_date_to", &filter.EndDateTo},
|
||||
} {
|
||||
v := strings.TrimSpace(r.URL.Query().Get(b.name))
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := time.Parse("2006-01-02", v); err != nil {
|
||||
httputil.WriteError(w, http.StatusBadRequest, b.name+" must be YYYY-MM-DD")
|
||||
return
|
||||
}
|
||||
*b.dst = &v
|
||||
}
|
||||
|
||||
if v := strings.TrimSpace(r.URL.Query().Get("file_type")); v != "" {
|
||||
ft := domain.FileType(v)
|
||||
@@ -320,10 +359,15 @@ type datasetPutRequest struct {
|
||||
Name string `json:"name" validate:"required,max=255"`
|
||||
Description *string `json:"description" validate:"required"`
|
||||
Unit *string `json:"unit" validate:"required"`
|
||||
// DgID, StartDate and EndDate are optional; omitting them (or sending null
|
||||
// or a blank string) clears the corresponding nullable column.
|
||||
DgID *string `json:"dg_id" validate:"omitempty,uuid"`
|
||||
StartDate *string `json:"start_date" validate:"omitempty,datetime=2006-01-02"`
|
||||
EndDate *string `json:"end_date" validate:"omitempty,datetime=2006-01-02"`
|
||||
}
|
||||
|
||||
// update replaces a dataset's editable metadata (category, name, description,
|
||||
// unit).
|
||||
// unit, dataset group, period).
|
||||
func (h *DatasetHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := parseUUIDParam(w, r, "id")
|
||||
if !ok {
|
||||
@@ -344,11 +388,24 @@ func (h *DatasetHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
var groupID *uuid.UUID
|
||||
if raw := optionalPtr(req.DgID); raw != nil {
|
||||
gid, err := uuid.Parse(*raw)
|
||||
if err != nil {
|
||||
httputil.WriteError(w, http.StatusUnprocessableEntity, "dg_id must be a valid UUID")
|
||||
return
|
||||
}
|
||||
groupID = &gid
|
||||
}
|
||||
|
||||
dataset, err := h.svc.Update(r.Context(), id, service.DatasetUpdateInput{
|
||||
CategoryID: categoryID,
|
||||
Name: strings.TrimSpace(req.Name),
|
||||
Description: normalizeOptional(*req.Description),
|
||||
Unit: normalizeOptional(*req.Unit),
|
||||
CategoryID: categoryID,
|
||||
Name: strings.TrimSpace(req.Name),
|
||||
Description: normalizeOptional(*req.Description),
|
||||
Unit: normalizeOptional(*req.Unit),
|
||||
DatasetGroupID: groupID,
|
||||
StartDate: optionalPtr(req.StartDate),
|
||||
EndDate: optionalPtr(req.EndDate),
|
||||
})
|
||||
if err != nil {
|
||||
respondDomainError(w, err)
|
||||
@@ -360,11 +417,13 @@ func (h *DatasetHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
// datasetPatchFields lists the JSON keys accepted by PATCH /datasets/{id}.
|
||||
var datasetPatchFields = map[string]struct{}{
|
||||
"category_id": {}, "name": {}, "description": {}, "unit": {},
|
||||
"dg_id": {}, "start_date": {}, "end_date": {},
|
||||
}
|
||||
|
||||
// patch applies a sparse update to a dataset's editable metadata (category,
|
||||
// name, description, unit): only fields present in the body are changed.
|
||||
// Sending an explicit null for description/unit clears that column.
|
||||
// name, description, unit, dg_id, start_date, end_date): only fields present in
|
||||
// the body are changed. Sending an explicit null for a nullable field clears
|
||||
// that column.
|
||||
func (h *DatasetHandler) patch(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := parseUUIDParam(w, r, "id")
|
||||
if !ok {
|
||||
@@ -421,6 +480,35 @@ func (h *DatasetHandler) patch(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
in.Unit, in.UnitSet = unit, true
|
||||
case "dg_id":
|
||||
raw, err := decodeOptionalPatchString(val)
|
||||
if err != nil {
|
||||
httputil.WriteError(w, http.StatusUnprocessableEntity, "dg_id must be a valid UUID or null")
|
||||
return
|
||||
}
|
||||
if raw != nil {
|
||||
gid, err := uuid.Parse(*raw)
|
||||
if err != nil {
|
||||
httputil.WriteError(w, http.StatusUnprocessableEntity, "dg_id must be a valid UUID or null")
|
||||
return
|
||||
}
|
||||
in.DatasetGroupID = &gid
|
||||
}
|
||||
in.DatasetGroupIDSet = true
|
||||
case "start_date":
|
||||
date, err := decodeOptionalPatchDate(val)
|
||||
if err != nil {
|
||||
httputil.WriteError(w, http.StatusUnprocessableEntity, "start_date must be YYYY-MM-DD or null")
|
||||
return
|
||||
}
|
||||
in.StartDate, in.StartDateSet = date, true
|
||||
case "end_date":
|
||||
date, err := decodeOptionalPatchDate(val)
|
||||
if err != nil {
|
||||
httputil.WriteError(w, http.StatusUnprocessableEntity, "end_date must be YYYY-MM-DD or null")
|
||||
return
|
||||
}
|
||||
in.EndDate, in.EndDateSet = date, true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -432,6 +520,15 @@ func (h *DatasetHandler) patch(w http.ResponseWriter, r *http.Request) {
|
||||
httputil.WriteJSON(w, http.StatusOK, dataset)
|
||||
}
|
||||
|
||||
// optionalPtr normalizes an optional request field: a nil or blank value
|
||||
// yields nil, so an omitted or empty field clears a nullable column.
|
||||
func optionalPtr(v *string) *string {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return normalizeOptional(*v)
|
||||
}
|
||||
|
||||
// normalizeOptional trims s and returns nil when blank, so an empty string
|
||||
// clears a nullable column rather than being stored verbatim.
|
||||
func normalizeOptional(s string) *string {
|
||||
@@ -455,6 +552,20 @@ func decodeOptionalPatchString(raw json.RawMessage) (*string, error) {
|
||||
return normalizeOptional(s), nil
|
||||
}
|
||||
|
||||
// decodeOptionalPatchDate unmarshals a PATCH date field that may be a
|
||||
// YYYY-MM-DD JSON string or null, returning nil (clear the column) for null or
|
||||
// a blank string.
|
||||
func decodeOptionalPatchDate(raw json.RawMessage) (*string, error) {
|
||||
v, err := decodeOptionalPatchString(raw)
|
||||
if err != nil || v == nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := time.Parse("2006-01-02", *v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// geojson returns the dataset's pre-assembled GeoJSON FeatureCollection (RFC
|
||||
// 7946), generated and stored at processing time. A vector_with_kato dataset's
|
||||
// collection joins the districts table on KATO code (one feature per KATO with
|
||||
|
||||
Reference in New Issue
Block a user