diff --git a/README.md b/README.md index 3f8402e..98b5609 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,9 @@ api/openapi.yaml OpenAPI 3.1.1 spec (embedded + served at /openapi.yaml) - **Category** — hierarchical (self-referencing `parent_id`). Full CRUD; cycle-safe on update. +- **DatasetGroup** — a named group of "одноимённые" datasets (see *Dataset + groups* below). Full CRUD; deleting a group still referenced by datasets is + rejected with 409. - **Dataset** — a geo file uploaded to S3/MinIO (`file_type`: `vector_with_kato | vector | raster`), belonging to one Category. Carries `code`/`name`/`description`/ `unit` metadata, a user-defined `meta` (JSONB) blob, an `automated` flag, a @@ -103,6 +106,26 @@ three are accepted on `POST /datasets` (form fields), `PUT /datasets/{id}` and `?start_date_from=`, `?start_date_to=`, `?end_date_from=` and `?end_date_to=` (each `YYYY-MM-DD`). +### Dataset groups + +A **dataset group** (`/dataset-groups`) collects the datasets that describe the +same indicator for different periods — the "одноимённые" datasets. A group has a +slug `code` (unique), a `name` and a `description`, and datasets join it through +their `dg_id`. + +**Freshness.** `GET /dataset-groups/{id}/latest` returns the group's freshest +dataset: the greatest `end_date` wins, datasets without an `end_date` come last, +and ties are broken by the newest `created_at`. Narrow the candidates with +`?status=` and `?file_type=` (for example `?status=ready&file_type=raster` to +get the latest published raster). A group with no matching dataset yields 404. + +**Shared SLD.** A group can carry one SLD (Styled Layer Descriptor) style file +(`POST`/`GET /dataset-groups/{id}/sld`, multipart `file`, `.sld` or `.xml`), +which acts as the fallback style for every dataset in the group: +`GET /datasets/{id}/sld` returns the dataset's own SLD when it has one, and +otherwise falls back to its group's — 404 only when neither exists. Deleting a +group also removes its stored SLD object. + ## Getting started ```sh @@ -138,7 +161,17 @@ server runs it is served at `/openapi.yaml`, with an interactive **Redoc** UI at | GET | `/datasets/{id}/download` | download the stored file | | POST | `/datasets/{id}/mapping` | set KATO column + year→period map (vector_with_kato) | | GET | `/datasets/{id}/observations` | paginated unpivoted values (`?kato_code=`, `?page=`, `?page_size=`) | +| POST | `/datasets/{id}/sld` | upload/replace the dataset's SLD (multipart `file`, `.sld`/`.xml`) | +| GET | `/datasets/{id}/sld` | download the dataset's SLD, falling back to its group's | | DELETE | `/datasets/{id}` | delete (row + object) | +| GET | `/dataset-groups` | paginated list (`?code=`, `?page=`, `?page_size=`) | +| POST | `/dataset-groups` | create (`code`, `name`, `description?`) | +| GET | `/dataset-groups/{id}` | get | +| PUT | `/dataset-groups/{id}` | update | +| DELETE | `/dataset-groups/{id}` | delete (409 while datasets reference it) | +| GET | `/dataset-groups/{id}/latest` | freshest dataset of the group (`?status=`, `?file_type=`) | +| POST | `/dataset-groups/{id}/sld` | upload/replace the group's shared SLD | +| GET | `/dataset-groups/{id}/sld` | download the group's shared SLD | Example upload: diff --git a/api/openapi.yaml b/api/openapi.yaml index 5cb1806..23c9b1a 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -33,6 +33,8 @@ tags: description: Hierarchical dataset categories - name: Datasets description: Geo dataset upload, processing, and retrieval + - name: Dataset Groups + description: Groups of same-named datasets versioned by period paths: /healthz: @@ -151,6 +153,165 @@ paths: "404": { $ref: "#/components/responses/NotFound" } "409": { $ref: "#/components/responses/Conflict" } + /dataset-groups: + get: + tags: [Dataset Groups] + summary: List dataset groups (paginated) + parameters: + - name: code + in: query + required: false + description: Filter by exact group code. + schema: + type: string + - $ref: "#/components/parameters/PageParam" + - $ref: "#/components/parameters/PageSizeParam" + responses: + "200": + description: Groups ordered by code + content: + application/json: + schema: { $ref: "#/components/schemas/DatasetGroupPage" } + "400": { $ref: "#/components/responses/BadRequest" } + post: + tags: [Dataset Groups] + summary: Create a dataset group + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/DatasetGroupInput" } + responses: + "201": + description: Created + content: + application/json: + schema: { $ref: "#/components/schemas/DatasetGroup" } + "400": { $ref: "#/components/responses/BadRequest" } + "409": { $ref: "#/components/responses/Conflict" } + "422": { $ref: "#/components/responses/ValidationError" } + + /dataset-groups/{id}: + parameters: + - $ref: "#/components/parameters/IdParam" + get: + tags: [Dataset Groups] + summary: Get a dataset group + responses: + "200": + description: The group + content: + application/json: + schema: { $ref: "#/components/schemas/DatasetGroup" } + "400": { $ref: "#/components/responses/BadRequest" } + "404": { $ref: "#/components/responses/NotFound" } + put: + tags: [Dataset Groups] + summary: Update a dataset group + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/DatasetGroupInput" } + responses: + "200": + description: Updated + content: + application/json: + schema: { $ref: "#/components/schemas/DatasetGroup" } + "400": { $ref: "#/components/responses/BadRequest" } + "404": { $ref: "#/components/responses/NotFound" } + "409": { $ref: "#/components/responses/Conflict" } + "422": { $ref: "#/components/responses/ValidationError" } + delete: + tags: [Dataset Groups] + summary: Delete a dataset group + description: Fails with 409 if datasets still reference the group. + responses: + "204": { description: Deleted } + "400": { $ref: "#/components/responses/BadRequest" } + "404": { $ref: "#/components/responses/NotFound" } + "409": { $ref: "#/components/responses/Conflict" } + + /dataset-groups/{id}/latest: + parameters: + - $ref: "#/components/parameters/IdParam" + get: + tags: [Dataset Groups] + summary: Get the freshest dataset of a group + description: | + Returns the group's dataset with the greatest `end_date` (datasets + without an `end_date` come last, ties broken by newest `created_at`). + Returns 404 when the group does not exist or has no matching dataset. + parameters: + - name: status + in: query + required: false + description: Only consider datasets in this status. + schema: { $ref: "#/components/schemas/DatasetStatus" } + - name: file_type + in: query + required: false + description: Only consider datasets of this file type. + schema: { $ref: "#/components/schemas/FileType" } + responses: + "200": + description: The freshest dataset in the group + content: + application/json: + schema: { $ref: "#/components/schemas/Dataset" } + "400": { $ref: "#/components/responses/BadRequest" } + "404": { $ref: "#/components/responses/NotFound" } + + /dataset-groups/{id}/sld: + parameters: + - $ref: "#/components/parameters/IdParam" + post: + tags: [Dataset Groups] + summary: Upload (or replace) a group's shared SLD style file + description: | + Multipart upload of an SLD (Styled Layer Descriptor) style file shared by + every dataset of the group. A dataset with no SLD of its own falls back + to this one on `GET /datasets/{id}/sld`. Uploading again replaces the + previously stored SLD. + + Allowed extensions: `.sld`, `.xml`. + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: [file] + properties: + file: + type: string + format: binary + description: The SLD file. + responses: + "200": + description: SLD stored; the updated group + content: + application/json: + schema: { $ref: "#/components/schemas/DatasetGroup" } + "400": { $ref: "#/components/responses/BadRequest" } + "404": { $ref: "#/components/responses/NotFound" } + "422": { $ref: "#/components/responses/ValidationError" } + get: + tags: [Dataset Groups] + summary: Download a group's shared SLD style file + description: Returns 404 if no SLD has been uploaded for this group yet. + responses: + "200": + description: The SLD file stream + content: + application/xml: + schema: + type: string + format: binary + "400": { $ref: "#/components/responses/BadRequest" } + "404": { $ref: "#/components/responses/NotFound" } + /datasets: get: tags: [Datasets] @@ -518,7 +679,10 @@ paths: get: tags: [Datasets] summary: Download a dataset's SLD style file - description: Returns 404 if no SLD has been uploaded for this dataset yet. + description: | + Returns the dataset's own SLD. When the dataset has none but belongs to a + dataset group that carries a shared SLD, the group's SLD is returned + instead. Returns 404 when neither exists. responses: "200": description: The SLD file stream @@ -676,6 +840,70 @@ components: description: Dataset processing lifecycle status. enum: [pending, processing, parsing, awaiting_mapping, extracting, ready, failed] + DatasetGroup: + type: object + description: | + A group of same-named datasets delivered for different periods. The + freshest member is the one with the greatest `end_date`. + required: [id, code, name, description, created_at, updated_at] + properties: + id: + type: string + format: uuid + code: + type: string + description: Slug (lowercase latin letters, digits, and dashes). + name: + type: string + description: + type: string + sld_storage_key: + type: [string, "null"] + description: Object-store key of the group's shared SLD, if uploaded. + sld_filename: + type: [string, "null"] + sld_content_type: + type: [string, "null"] + created_at: + type: string + format: date-time + updated_at: + type: string + format: date-time + + DatasetGroupInput: + type: object + required: [code, name] + properties: + code: + type: string + maxLength: 255 + pattern: "^[a-z0-9]+(?:-[a-z0-9]+)*$" + description: Slug (lowercase latin letters, digits, and dashes). + name: + type: string + maxLength: 255 + description: + type: string + maxLength: 2000 + + DatasetGroupPage: + type: object + required: [data, page, page_size, total, total_pages] + properties: + data: + type: array + items: + $ref: "#/components/schemas/DatasetGroup" + page: + type: integer + page_size: + type: integer + total: + type: integer + total_pages: + type: integer + Category: type: object required: [id, code, name, description, created_at, updated_at] diff --git a/internal/app/app.go b/internal/app/app.go index eb12004..281eff2 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -38,6 +38,7 @@ type App struct { publisher *rabbitmq.Publisher categories *service.CategoryService datasets *service.DatasetService + groups *service.DatasetGroupService eventRepo *postgres.EventRepository } @@ -68,6 +69,7 @@ func New(ctx context.Context) (*App, error) { categoryRepo := postgres.NewCategoryRepository(pool) datasetRepo := postgres.NewDatasetRepository(pool) + groupRepo := postgres.NewDatasetGroupRepository(pool) eventRepo := postgres.NewEventRepository(pool) publisher := rabbitmq.NewPublisher(rabbit) @@ -81,7 +83,8 @@ func New(ctx context.Context) (*App, error) { rabbit: rabbit, publisher: publisher, categories: service.NewCategoryService(categoryRepo), - datasets: service.NewDatasetService(datasetRepo, store, categoryRepo, jobPublisher, parser.Columns, parser.Rows, raster.NewGDALConverter()), + datasets: service.NewDatasetService(datasetRepo, store, categoryRepo, groupRepo, jobPublisher, parser.Columns, parser.Rows, raster.NewGDALConverter()), + groups: service.NewDatasetGroupService(groupRepo, datasetRepo, store), eventRepo: eventRepo, }, nil } @@ -100,11 +103,12 @@ func (a *App) Handler() stdhttp.Handler { }) return transporthttp.NewRouter(transporthttp.RouterDeps{ - Logger: a.Log, - Health: health, - Categories: transporthttp.NewCategoryHandler(a.categories, validate), - Datasets: transporthttp.NewDatasetHandler(a.datasets, validate), - OpenAPISpec: api.Spec, + Logger: a.Log, + Health: health, + Categories: transporthttp.NewCategoryHandler(a.categories, validate), + Datasets: transporthttp.NewDatasetHandler(a.datasets, validate), + DatasetGroups: transporthttp.NewDatasetGroupHandler(a.groups, validate), + OpenAPISpec: api.Spec, }) } diff --git a/internal/domain/dataset_group.go b/internal/domain/dataset_group.go new file mode 100644 index 0000000..3739773 --- /dev/null +++ b/internal/domain/dataset_group.go @@ -0,0 +1,23 @@ +package domain + +import ( + "time" + + "github.com/google/uuid" +) + +// DatasetGroup groups "одноимённые" datasets (the same indicator delivered for +// different periods). The freshest dataset of a group is the one with the +// greatest end_date. A group may carry a shared SLD style used as a fallback by +// datasets that have none of their own. +type DatasetGroup struct { + ID uuid.UUID `json:"id"` + Code string `json:"code"` + Name string `json:"name"` + Description string `json:"description"` + SldStorageKey *string `json:"sld_storage_key"` + SldFilename *string `json:"sld_filename"` + SldContentType *string `json:"sld_content_type"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/internal/repository/postgres/dataset_group.go b/internal/repository/postgres/dataset_group.go new file mode 100644 index 0000000..de24b8a --- /dev/null +++ b/internal/repository/postgres/dataset_group.go @@ -0,0 +1,117 @@ +package postgres + +import ( + "context" + "fmt" + + "gis/internal/domain" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// DatasetGroupRepository persists dataset groups in Postgres. +type DatasetGroupRepository struct{ pool *pgxpool.Pool } + +// NewDatasetGroupRepository returns a DatasetGroupRepository backed by pool. +func NewDatasetGroupRepository(pool *pgxpool.Pool) *DatasetGroupRepository { + return &DatasetGroupRepository{pool: pool} +} + +const datasetGroupColumns = `id, code, name, description, sld_storage_key, sld_filename, sld_content_type, created_at, updated_at` + +func scanDatasetGroup(row pgx.Row) (domain.DatasetGroup, error) { + var g domain.DatasetGroup + err := row.Scan(&g.ID, &g.Code, &g.Name, &g.Description, &g.SldStorageKey, &g.SldFilename, &g.SldContentType, &g.CreatedAt, &g.UpdatedAt) + return g, err +} + +// Create inserts a new dataset group and returns the stored row. +func (r *DatasetGroupRepository) Create(ctx context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) { + row := r.pool.QueryRow(ctx, + `INSERT INTO dataset_groups (code, name, description) VALUES ($1, $2, $3) RETURNING `+datasetGroupColumns, + g.Code, g.Name, g.Description) + out, err := scanDatasetGroup(row) + return out, mapError(err) +} + +// GetByID returns the group with the given id. +func (r *DatasetGroupRepository) GetByID(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, error) { + out, err := scanDatasetGroup(r.pool.QueryRow(ctx, `SELECT `+datasetGroupColumns+` FROM dataset_groups WHERE id = $1`, id)) + return out, mapError(err) +} + +// GetByCode returns the group with the given (unique) code. +func (r *DatasetGroupRepository) GetByCode(ctx context.Context, code string) (domain.DatasetGroup, error) { + out, err := scanDatasetGroup(r.pool.QueryRow(ctx, `SELECT `+datasetGroupColumns+` FROM dataset_groups WHERE code = $1`, code)) + return out, mapError(err) +} + +// List returns a page of groups ordered by code; code (exact) is optional. +func (r *DatasetGroupRepository) List(ctx context.Context, code *string, limit, offset int) ([]domain.DatasetGroup, error) { + where, args := groupWhere(code) + args = append(args, limit, offset) + rows, err := r.pool.Query(ctx, fmt.Sprintf(`SELECT %s FROM dataset_groups%s ORDER BY code LIMIT $%d OFFSET $%d`, + datasetGroupColumns, where, len(args)-1, len(args)), args...) + if err != nil { + return nil, mapError(err) + } + defer rows.Close() + out := make([]domain.DatasetGroup, 0) + for rows.Next() { + g, err := scanDatasetGroup(rows) + if err != nil { + return nil, mapError(err) + } + out = append(out, g) + } + return out, mapError(rows.Err()) +} + +// Count returns how many groups match the optional code filter. +func (r *DatasetGroupRepository) Count(ctx context.Context, code *string) (int, error) { + where, args := groupWhere(code) + var n int + err := r.pool.QueryRow(ctx, `SELECT count(*) FROM dataset_groups`+where, args...).Scan(&n) + return n, mapError(err) +} + +// groupWhere builds the optional exact-code WHERE clause and its arguments. +func groupWhere(code *string) (string, []any) { + if code == nil { + return "", nil + } + return " WHERE code = $1", []any{*code} +} + +// Update replaces a group's editable fields and returns the stored row. +func (r *DatasetGroupRepository) Update(ctx context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) { + row := r.pool.QueryRow(ctx, + `UPDATE dataset_groups SET code = $2, name = $3, description = $4, updated_at = now() WHERE id = $1 RETURNING `+datasetGroupColumns, + g.ID, g.Code, g.Name, g.Description) + out, err := scanDatasetGroup(row) + return out, mapError(err) +} + +// Delete removes a group. Returns domain.ErrNotFound if absent and +// domain.ErrConflict (via the FK) if datasets still reference it. +func (r *DatasetGroupRepository) Delete(ctx context.Context, id uuid.UUID) error { + tag, err := r.pool.Exec(ctx, `DELETE FROM dataset_groups WHERE id = $1`, id) + if err != nil { + return mapError(err) + } + if tag.RowsAffected() == 0 { + return domain.ErrNotFound + } + return nil +} + +// SetSLD points the group at a stored SLD style object. +func (r *DatasetGroupRepository) SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.DatasetGroup, error) { + row := r.pool.QueryRow(ctx, + `UPDATE dataset_groups SET sld_storage_key = $2, sld_filename = $3, sld_content_type = $4, updated_at = now() WHERE id = $1 RETURNING `+datasetGroupColumns, + id, key, filename, contentType) + out, err := scanDatasetGroup(row) + return out, mapError(err) +} diff --git a/internal/service/dataset.go b/internal/service/dataset.go index 746feb5..640c6b7 100644 --- a/internal/service/dataset.go +++ b/internal/service/dataset.go @@ -86,6 +86,12 @@ type categoryReader interface { GetByCode(ctx context.Context, code string) (domain.Category, error) } +// groupReader lets the dataset service verify a dataset group exists before a +// dataset is filed under it, and resolve its shared SLD as a fallback. +type groupReader interface { + GetByID(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, error) +} + // JobEnqueuer schedules asynchronous dataset jobs. type JobEnqueuer interface { EnqueueParse(ctx context.Context, datasetID uuid.UUID) error @@ -138,6 +144,7 @@ type DatasetService struct { repo DatasetRepository store ObjectStore categories categoryReader + groups groupReader jobs JobEnqueuer parseColumns ColumnParser parseRows RowParser @@ -145,12 +152,14 @@ type DatasetService struct { } // NewDatasetService wires the dataset repository, object store, category reader -// (for parent validation), the job enqueuer, the column/row parsers, and the -// raster converter. +// (for parent validation), the dataset-group reader (for group validation and +// the SLD fallback), the job enqueuer, the column/row parsers, and the raster +// converter. func NewDatasetService( repo DatasetRepository, store ObjectStore, categories categoryReader, + groups groupReader, jobs JobEnqueuer, parseColumns ColumnParser, parseRows RowParser, @@ -160,6 +169,7 @@ func NewDatasetService( repo: repo, store: store, categories: categories, + groups: groups, jobs: jobs, parseColumns: parseColumns, parseRows: parseRows, @@ -196,6 +206,9 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat if err := s.ensureCategoryExists(ctx, in.CategoryID); err != nil { return domain.Dataset{}, err } + if err := s.ensureGroupExists(ctx, in.DatasetGroupID); err != nil { + return domain.Dataset{}, err + } if err := domain.ValidateDatePeriod(in.StartDate, in.EndDate); err != nil { return domain.Dataset{}, err } @@ -311,7 +324,23 @@ func (s *DatasetService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain. return domain.Dataset{}, nil, err } if dataset.SldStorageKey == nil { - return domain.Dataset{}, nil, ErrSLDNotFound + // Fall back to the dataset group's shared style, if the group has one. + if dataset.DatasetGroupID == nil { + return domain.Dataset{}, nil, ErrSLDNotFound + } + group, err := s.groups.GetByID(ctx, *dataset.DatasetGroupID) + if err != nil { + if errors.Is(err, domain.ErrNotFound) { + return domain.Dataset{}, nil, ErrSLDNotFound + } + return domain.Dataset{}, nil, err + } + if group.SldStorageKey == nil { + return domain.Dataset{}, nil, ErrSLDNotFound + } + dataset.SldStorageKey = group.SldStorageKey + dataset.SldFilename = group.SldFilename + dataset.SldContentType = group.SldContentType } obj, err := s.store.Get(ctx, *dataset.SldStorageKey) if err != nil { @@ -320,6 +349,48 @@ func (s *DatasetService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain. return dataset, obj, nil } +// CreateRasterInput describes a raster file already on local disk that should +// become a dataset (used by the Landscape Program to publish results). +type CreateRasterInput struct { + CategoryID uuid.UUID + DatasetGroupID *uuid.UUID + Name string + Filename string // stored filename, e.g. "lp-z-integrated.tif" + Path string // local path to read from + StartDate *string + EndDate *string + Meta json.RawMessage + Automated bool +} + +// CreateRasterFromFile stores a local GeoTIFF as a new raster dataset and +// enqueues the standard COG conversion, exactly like an uploaded raster. +func (s *DatasetService) CreateRasterFromFile(ctx context.Context, in CreateRasterInput) (domain.Dataset, error) { + f, err := os.Open(in.Path) + if err != nil { + return domain.Dataset{}, err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return domain.Dataset{}, err + } + return s.Upload(ctx, UploadInput{ + CategoryID: in.CategoryID, + DatasetGroupID: in.DatasetGroupID, + Name: in.Name, + Meta: in.Meta, + Automated: in.Automated, + Filename: in.Filename, + FileType: domain.FileTypeRaster, + ContentType: "image/tiff", + Size: info.Size(), + Reader: f, + StartDate: in.StartDate, + EndDate: in.EndDate, + }) +} + // deriveSLDKey places a dataset's SLD style file alongside the original under // an sld/ prefix, named after the uploaded SLD file. func deriveSLDKey(storageKey, filename string) string { @@ -1099,6 +1170,9 @@ func (s *DatasetService) Update(ctx context.Context, id uuid.UUID, in DatasetUpd if err := s.ensureCategoryExists(ctx, in.CategoryID); err != nil { return domain.Dataset{}, err } + if err := s.ensureGroupExists(ctx, in.DatasetGroupID); err != nil { + return domain.Dataset{}, err + } if err := domain.ValidateDatePeriod(in.StartDate, in.EndDate); err != nil { return domain.Dataset{}, err } @@ -1163,6 +1237,9 @@ func (s *DatasetService) Patch(ctx context.Context, id uuid.UUID, in DatasetPatc groupID := current.DatasetGroupID if in.DatasetGroupIDSet { groupID = in.DatasetGroupID + if err := s.ensureGroupExists(ctx, groupID); err != nil { + return domain.Dataset{}, err + } } startDate := current.StartDate if in.StartDateSet { @@ -1199,6 +1276,21 @@ func (s *DatasetService) ensureCategoryExists(ctx context.Context, categoryID uu return nil } +// ensureGroupExists validates that a non-nil dataset group id references an +// existing group. A nil id (no group) is always valid. +func (s *DatasetService) ensureGroupExists(ctx context.Context, id *uuid.UUID) error { + if id == nil { + return nil + } + if _, err := s.groups.GetByID(ctx, *id); err != nil { + if errors.Is(err, domain.ErrNotFound) { + return fmt.Errorf("%w: dataset group %s not found", domain.ErrValidation, id) + } + return err + } + return nil +} + // Status-wait bounds and polling cadence for long polling. const ( DefaultStatusWait = 25 * time.Second diff --git a/internal/service/dataset_group.go b/internal/service/dataset_group.go new file mode 100644 index 0000000..b833757 --- /dev/null +++ b/internal/service/dataset_group.go @@ -0,0 +1,185 @@ +package service + +import ( + "bytes" + "context" + "fmt" + "io" + "path" + "path/filepath" + "strings" + + "gis/internal/domain" + + "github.com/google/uuid" +) + +// DatasetGroupRepository is the persistence behaviour DatasetGroupService needs. +type DatasetGroupRepository interface { + Create(ctx context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) + GetByID(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, error) + GetByCode(ctx context.Context, code string) (domain.DatasetGroup, error) + List(ctx context.Context, code *string, limit, offset int) ([]domain.DatasetGroup, error) + Count(ctx context.Context, code *string) (int, error) + Update(ctx context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) + Delete(ctx context.Context, id uuid.UUID) error + SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.DatasetGroup, error) +} + +// DatasetGroupInput carries the mutable fields of a group. +type DatasetGroupInput struct { + Code string + Name string + Description string +} + +// DatasetGroupPage is a page of groups. +type DatasetGroupPage struct { + Items []domain.DatasetGroup + Page int + PageSize int + Total int +} + +// DatasetGroupService implements dataset-group business rules. +type DatasetGroupService struct { + repo DatasetGroupRepository + datasets DatasetRepository + store ObjectStore +} + +// NewDatasetGroupService wires the group repository, the dataset repository +// (for the "latest" lookup) and the object store (for the group SLD). +func NewDatasetGroupService(repo DatasetGroupRepository, datasets DatasetRepository, store ObjectStore) *DatasetGroupService { + return &DatasetGroupService{repo: repo, datasets: datasets, store: store} +} + +// errInvalidGroupCode is the validation error returned for a non-slug code. +func errInvalidGroupCode() error { + return fmt.Errorf("%w: code must be a slug (lowercase latin letters, digits, and dashes)", domain.ErrValidation) +} + +// Create persists a new group after validating its code. +func (s *DatasetGroupService) Create(ctx context.Context, in DatasetGroupInput) (domain.DatasetGroup, error) { + if !domain.ValidSlug(in.Code) { + return domain.DatasetGroup{}, errInvalidGroupCode() + } + return s.repo.Create(ctx, domain.DatasetGroup{Code: in.Code, Name: in.Name, Description: in.Description}) +} + +// Get returns one group by id. +func (s *DatasetGroupService) Get(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, error) { + return s.repo.GetByID(ctx, id) +} + +// List returns a page of groups; page/pageSize are clamped like datasets. +func (s *DatasetGroupService) List(ctx context.Context, code *string, page, pageSize int) (DatasetGroupPage, error) { + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = DefaultPageSize + } + if pageSize > MaxPageSize { + pageSize = MaxPageSize + } + items, err := s.repo.List(ctx, code, pageSize, (page-1)*pageSize) + if err != nil { + return DatasetGroupPage{}, err + } + total, err := s.repo.Count(ctx, code) + if err != nil { + return DatasetGroupPage{}, err + } + return DatasetGroupPage{Items: items, Page: page, PageSize: pageSize, Total: total}, nil +} + +// Update replaces a group's editable fields. +func (s *DatasetGroupService) Update(ctx context.Context, id uuid.UUID, in DatasetGroupInput) (domain.DatasetGroup, error) { + if !domain.ValidSlug(in.Code) { + return domain.DatasetGroup{}, errInvalidGroupCode() + } + return s.repo.Update(ctx, domain.DatasetGroup{ID: id, Code: in.Code, Name: in.Name, Description: in.Description}) +} + +// Delete removes a group (409 via ErrConflict while datasets reference it) and +// then best-effort removes its SLD object. +func (s *DatasetGroupService) Delete(ctx context.Context, id uuid.UUID) error { + g, err := s.repo.GetByID(ctx, id) + if err != nil { + return err + } + if err := s.repo.Delete(ctx, id); err != nil { + return err + } + if g.SldStorageKey != nil { + _ = s.store.Remove(ctx, *g.SldStorageKey) // best-effort: the row is gone + } + return nil +} + +// Latest returns the freshest dataset of the group (greatest end_date, NULLs +// last, then newest), optionally restricted by status and file type. +func (s *DatasetGroupService) Latest(ctx context.Context, id uuid.UUID, status *string, fileType *domain.FileType) (domain.Dataset, error) { + if _, err := s.repo.GetByID(ctx, id); err != nil { + return domain.Dataset{}, err + } + return s.datasets.LatestInGroup(ctx, id, status, fileType) +} + +// UploadSLD validates and stores the group's shared SLD, replacing any previous one. +func (s *DatasetGroupService) UploadSLD(ctx context.Context, id uuid.UUID, in UploadSLDInput) (domain.DatasetGroup, error) { + g, err := s.repo.GetByID(ctx, id) + if err != nil { + return domain.DatasetGroup{}, err + } + + ext := strings.ToLower(filepath.Ext(in.Filename)) + if !domain.ExtensionAllowedForSLD(ext) { + return domain.DatasetGroup{}, fmt.Errorf("%w: sld file must have one of these extensions: %s", + domain.ErrValidation, strings.Join(domain.AllowedSLDExtensions, ", ")) + } + + head := make([]byte, 512) + n, err := io.ReadFull(in.Reader, head) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return domain.DatasetGroup{}, fmt.Errorf("read sld upload: %w", err) + } + head = head[:n] + if err := domain.ValidateFileContent(ext, head); err != nil { + return domain.DatasetGroup{}, fmt.Errorf("%w: %s", domain.ErrValidation, err) + } + content := io.MultiReader(bytes.NewReader(head), in.Reader) + + key := path.Join("groups", g.ID.String(), "sld", in.Filename) + if err := s.store.Put(ctx, key, content, in.Size, in.ContentType); err != nil { + return domain.DatasetGroup{}, err + } + updated, err := s.repo.SetSLD(ctx, id, key, in.Filename, in.ContentType) + if err != nil { + // Compensate: the row was not updated, so the object would be orphaned. + _ = s.store.Remove(ctx, key) + return domain.DatasetGroup{}, err + } + if g.SldStorageKey != nil && *g.SldStorageKey != key { + _ = s.store.Remove(ctx, *g.SldStorageKey) // best-effort: drop the replaced file + } + return updated, nil +} + +// DownloadSLD returns the group and a reader for its SLD; ErrSLDNotFound when +// none has been uploaded. The caller must close the reader. +func (s *DatasetGroupService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, io.ReadCloser, error) { + g, err := s.repo.GetByID(ctx, id) + if err != nil { + return domain.DatasetGroup{}, nil, err + } + if g.SldStorageKey == nil { + return domain.DatasetGroup{}, nil, ErrSLDNotFound + } + obj, err := s.store.Get(ctx, *g.SldStorageKey) + if err != nil { + return domain.DatasetGroup{}, nil, err + } + return g, obj, nil +} diff --git a/internal/service/dataset_group_test.go b/internal/service/dataset_group_test.go new file mode 100644 index 0000000..373e3b1 --- /dev/null +++ b/internal/service/dataset_group_test.go @@ -0,0 +1,257 @@ +package service + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "gis/internal/domain" + + "github.com/google/uuid" +) + +// stubGroupRepo is an in-memory DatasetGroupRepository for tests. It also +// satisfies the dataset service's groupReader dependency. +type stubGroupRepo struct { + store map[uuid.UUID]domain.DatasetGroup +} + +func newStubGroupRepo() *stubGroupRepo { + return &stubGroupRepo{store: map[uuid.UUID]domain.DatasetGroup{}} +} + +func (r *stubGroupRepo) Create(_ context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) { + for _, e := range r.store { + if e.Code == g.Code { + return domain.DatasetGroup{}, domain.ErrConflict + } + } + g.ID = uuid.New() + r.store[g.ID] = g + return g, nil +} + +func (r *stubGroupRepo) GetByID(_ context.Context, id uuid.UUID) (domain.DatasetGroup, error) { + g, ok := r.store[id] + if !ok { + return domain.DatasetGroup{}, domain.ErrNotFound + } + return g, nil +} + +func (r *stubGroupRepo) GetByCode(_ context.Context, code string) (domain.DatasetGroup, error) { + for _, g := range r.store { + if g.Code == code { + return g, nil + } + } + return domain.DatasetGroup{}, domain.ErrNotFound +} + +func (r *stubGroupRepo) List(_ context.Context, code *string, _, _ int) ([]domain.DatasetGroup, error) { + out := []domain.DatasetGroup{} + for _, g := range r.store { + if code == nil || g.Code == *code { + out = append(out, g) + } + } + return out, nil +} + +func (r *stubGroupRepo) Count(_ context.Context, code *string) (int, error) { + l, _ := r.List(context.Background(), code, 0, 0) + return len(l), nil +} + +func (r *stubGroupRepo) Update(_ context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) { + if _, ok := r.store[g.ID]; !ok { + return domain.DatasetGroup{}, domain.ErrNotFound + } + r.store[g.ID] = g + return g, nil +} + +func (r *stubGroupRepo) Delete(_ context.Context, id uuid.UUID) error { + if _, ok := r.store[id]; !ok { + return domain.ErrNotFound + } + delete(r.store, id) + return nil +} + +func (r *stubGroupRepo) SetSLD(_ context.Context, id uuid.UUID, key, filename, contentType string) (domain.DatasetGroup, error) { + g, ok := r.store[id] + if !ok { + return domain.DatasetGroup{}, domain.ErrNotFound + } + g.SldStorageKey, g.SldFilename, g.SldContentType = &key, &filename, &contentType + r.store[id] = g + return g, nil +} + +func TestDatasetGroupService_Create_RequiresSlug(t *testing.T) { + svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore()) + _, err := svc.Create(context.Background(), DatasetGroupInput{Code: "Bad Code", Name: "x"}) + if !errors.Is(err, domain.ErrValidation) { + t.Fatalf("want ErrValidation, got %v", err) + } +} + +func TestDatasetGroupService_Latest_PrefersGreatestEndDateThenCreatedAt(t *testing.T) { + groups := newStubGroupRepo() + datasets := newStubDatasetRepo() + svc := NewDatasetGroupService(groups, datasets, newStubStore()) + g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"}) + if err != nil { + t.Fatal(err) + } + s := func(v string) *string { return &v } + base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + old := domain.Dataset{ID: uuid.New(), DatasetGroupID: &g.ID, EndDate: s("2019-12-31"), Status: domain.DatasetStatusReady, CreatedAt: base} + fresh := domain.Dataset{ID: uuid.New(), DatasetGroupID: &g.ID, EndDate: s("2020-12-31"), Status: domain.DatasetStatusReady, CreatedAt: base.Add(time.Hour)} + undated := domain.Dataset{ID: uuid.New(), DatasetGroupID: &g.ID, Status: domain.DatasetStatusReady, CreatedAt: base.Add(2 * time.Hour)} + for _, d := range []domain.Dataset{old, fresh, undated} { + datasets.store[d.ID] = d + } + got, err := svc.Latest(context.Background(), g.ID, nil, nil) + if err != nil { + t.Fatal(err) + } + if got.ID != fresh.ID { + t.Fatalf("want %s, got %s", fresh.ID, got.ID) + } +} + +func TestDatasetGroupService_Latest_NotFoundWhenEmpty(t *testing.T) { + groups := newStubGroupRepo() + svc := NewDatasetGroupService(groups, newStubDatasetRepo(), newStubStore()) + g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"}) + if err != nil { + t.Fatal(err) + } + if _, err := svc.Latest(context.Background(), g.ID, nil, nil); !errors.Is(err, domain.ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestDatasetGroupService_Latest_UnknownGroupIsNotFound(t *testing.T) { + svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore()) + if _, err := svc.Latest(context.Background(), uuid.New(), nil, nil); !errors.Is(err, domain.ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestDatasetGroupService_UploadSLD_RejectsBadExtension(t *testing.T) { + groups := newStubGroupRepo() + store := newStubStore() + svc := NewDatasetGroupService(groups, newStubDatasetRepo(), store) + g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"}) + if err != nil { + t.Fatal(err) + } + _, err = svc.UploadSLD(context.Background(), g.ID, UploadSLDInput{Filename: "style.txt", ContentType: "text/plain", Size: 3, Reader: strings.NewReader("")}) + if !errors.Is(err, domain.ErrValidation) { + t.Fatalf("want ErrValidation, got %v", err) + } + if len(store.put) != 0 { + t.Fatalf("nothing should be stored on validation failure, got %v", store.put) + } +} + +func TestDatasetGroupService_UploadSLD_StoresAndReplaces(t *testing.T) { + ctx := context.Background() + groups := newStubGroupRepo() + store := newStubStore() + svc := NewDatasetGroupService(groups, newStubDatasetRepo(), store) + g, err := svc.Create(ctx, DatasetGroupInput{Code: "pop", Name: "Population"}) + if err != nil { + t.Fatal(err) + } + + updated, err := svc.UploadSLD(ctx, g.ID, UploadSLDInput{ + Filename: "style.sld", ContentType: "application/xml", Size: 3, Reader: strings.NewReader(""), + }) + if err != nil { + t.Fatal(err) + } + wantKey := "groups/" + g.ID.String() + "/sld/style.sld" + if updated.SldStorageKey == nil || *updated.SldStorageKey != wantKey { + t.Fatalf("want key %s, got %v", wantKey, updated.SldStorageKey) + } + + // A second upload under a different name replaces the previous object. + if _, err := svc.UploadSLD(ctx, g.ID, UploadSLDInput{ + Filename: "other.sld", ContentType: "application/xml", Size: 3, Reader: strings.NewReader(""), + }); err != nil { + t.Fatal(err) + } + if len(store.removed) != 1 || store.removed[0] != wantKey { + t.Fatalf("want previous sld removed, got %v", store.removed) + } +} + +func TestDatasetGroupService_DownloadSLD_NotFoundWhenAbsent(t *testing.T) { + svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore()) + g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"}) + if err != nil { + t.Fatal(err) + } + if _, _, err := svc.DownloadSLD(context.Background(), g.ID); !errors.Is(err, ErrSLDNotFound) { + t.Fatalf("want ErrSLDNotFound, got %v", err) + } +} + +func TestDatasetGroupService_Delete_RemovesSLDObject(t *testing.T) { + ctx := context.Background() + store := newStubStore() + svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), store) + g, err := svc.Create(ctx, DatasetGroupInput{Code: "pop", Name: "Population"}) + if err != nil { + t.Fatal(err) + } + if _, err := svc.UploadSLD(ctx, g.ID, UploadSLDInput{ + Filename: "style.sld", ContentType: "application/xml", Size: 3, Reader: strings.NewReader(""), + }); err != nil { + t.Fatal(err) + } + if err := svc.Delete(ctx, g.ID); err != nil { + t.Fatal(err) + } + wantKey := "groups/" + g.ID.String() + "/sld/style.sld" + if len(store.removed) != 1 || store.removed[0] != wantKey { + t.Fatalf("want sld object removed, got %v", store.removed) + } + if _, err := svc.Get(ctx, g.ID); !errors.Is(err, domain.ErrNotFound) { + t.Fatalf("want ErrNotFound after delete, got %v", err) + } +} + +func TestDatasetGroupService_List_ClampsPageSize(t *testing.T) { + svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore()) + if _, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"}); err != nil { + t.Fatal(err) + } + page, err := svc.List(context.Background(), nil, 0, 10_000) + if err != nil { + t.Fatal(err) + } + if page.Page != 1 || page.PageSize != MaxPageSize { + t.Fatalf("want page 1 and page size %d, got %d/%d", MaxPageSize, page.Page, page.PageSize) + } + if page.Total != 1 { + t.Fatalf("want total 1, got %d", page.Total) + } +} + +func TestDatasetGroupService_Update_RequiresSlug(t *testing.T) { + svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore()) + g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"}) + if err != nil { + t.Fatal(err) + } + if _, err := svc.Update(context.Background(), g.ID, DatasetGroupInput{Code: "Bad Code", Name: "x"}); !errors.Is(err, domain.ErrValidation) { + t.Fatalf("want ErrValidation, got %v", err) + } +} diff --git a/internal/service/dataset_test.go b/internal/service/dataset_test.go index 5ac2aa5..b6f9dde 100644 --- a/internal/service/dataset_test.go +++ b/internal/service/dataset_test.go @@ -7,6 +7,7 @@ import ( "errors" "io" "os" + "path/filepath" "sort" "strings" "testing" @@ -358,22 +359,36 @@ var ( noopRowParser RowParser = func(string, []byte) ([]map[string]string, error) { return nil, nil } ) -// stubStore records object-store interactions. +// stubStore records object-store interactions and keeps stored bodies so tests +// can read them back. type stubStore struct { put []string removed []string putErr error + objects map[string][]byte } -func (s *stubStore) Put(_ context.Context, key string, _ io.Reader, _ int64, _ string) error { +func newStubStore() *stubStore { return &stubStore{objects: map[string][]byte{}} } + +func (s *stubStore) Put(_ context.Context, key string, r io.Reader, _ int64, _ string) error { if s.putErr != nil { return s.putErr } s.put = append(s.put, key) + if s.objects != nil { + body, err := io.ReadAll(r) + if err != nil { + return err + } + s.objects[key] = body + } return nil } -func (s *stubStore) Get(_ context.Context, _ string) (io.ReadCloser, error) { +func (s *stubStore) Get(_ context.Context, key string) (io.ReadCloser, error) { + if body, ok := s.objects[key]; ok { + return io.NopCloser(bytes.NewReader(body)), nil + } return io.NopCloser(strings.NewReader("")), nil } @@ -414,7 +429,13 @@ func validUpload() UploadInput { } func newDatasetService(repo *stubDatasetRepo, store *stubStore, catExists bool) *DatasetService { - return NewDatasetService(repo, store, stubCategoryReader{exists: catExists}, &stubEnqueuer{}, noopParser, noopRowParser, &stubConverter{}) + return newDatasetServiceWithGroups(repo, store, catExists, newStubGroupRepo()) +} + +// newDatasetServiceWithGroups builds a DatasetService over a caller-supplied +// group repository, for tests that need a pre-populated dataset group. +func newDatasetServiceWithGroups(repo *stubDatasetRepo, store *stubStore, catExists bool, groups *stubGroupRepo) *DatasetService { + return NewDatasetService(repo, store, stubCategoryReader{exists: catExists}, groups, &stubEnqueuer{}, noopParser, noopRowParser, &stubConverter{}) } func TestDatasetService_Upload_Validation(t *testing.T) { @@ -521,7 +542,7 @@ func TestDatasetService_Upload_CompensatesOnDBFailure(t *testing.T) { func TestDatasetService_Upload_VectorWithKato_EnqueuesParse(t *testing.T) { repo := newStubDatasetRepo() enq := &stubEnqueuer{} - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{}) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{}) in := validUpload() in.FileType = domain.FileTypeVectorWithKato @@ -542,7 +563,7 @@ func TestDatasetService_Upload_VectorWithKato_EnqueuesParse(t *testing.T) { func TestDatasetService_Upload_Vector_EnqueuesProperties(t *testing.T) { repo := newStubDatasetRepo() enq := &stubEnqueuer{} - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{}) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{}) got, err := svc.Upload(context.Background(), validUpload()) // plain vector .geojson if err != nil { @@ -573,7 +594,7 @@ func TestDatasetService_ExtractProperties(t *testing.T) { json.RawMessage(`{"type":"Polygon","coordinates":[[[4,4],[5,5],[6,4],[4,4]]]}`), } conv := &stubConverter{vectorFeatures: geoms} - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, conv) if err := svc.ExtractProperties(context.Background(), id); err != nil { t.Fatalf("unexpected error: %v", err) @@ -625,7 +646,7 @@ func TestDatasetService_ExtractProperties_GeometryBestEffort(t *testing.T) { conv := &stubConverter{vectorFeaturesFn: func(string) ([]json.RawMessage, error) { return nil, errors.New("ogr2ogr boom") }} - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, conv) if err := svc.ExtractProperties(context.Background(), id); err != nil { t.Fatalf("unexpected error: %v", err) @@ -653,7 +674,7 @@ func TestDatasetService_ExtractProperties_NoTable(t *testing.T) { rp := RowParser(func(string, []byte) ([]map[string]string, error) { return []map[string]string{{}, {}}, nil }) - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, &stubConverter{}) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, &stubConverter{}) if err := svc.ExtractProperties(context.Background(), id); err != nil { t.Fatalf("unexpected error: %v", err) @@ -670,7 +691,7 @@ func TestDatasetService_ExtractProperties_NoTable(t *testing.T) { func TestDatasetService_Upload_Raster_EnqueuesConvert(t *testing.T) { repo := newStubDatasetRepo() enq := &stubEnqueuer{} - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{}) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{}) in := validUpload() in.FileType = domain.FileTypeRaster @@ -700,7 +721,7 @@ func TestDatasetService_ConvertToCOG(t *testing.T) { store := &stubStore{} footprint := []byte(`{"type":"Polygon","coordinates":[[[70,50],[72,50],[72,52],[70,52],[70,50]]]}`) conv := &stubConverter{footprint: footprint} - svc := NewDatasetService(repo, store, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, noopRowParser, conv) + svc := NewDatasetService(repo, store, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, noopRowParser, conv) if err := svc.ConvertToCOG(context.Background(), id); err != nil { t.Fatalf("unexpected error: %v", err) @@ -732,7 +753,7 @@ func TestDatasetService_ConvertToCOG_RecordsFailure(t *testing.T) { Status: domain.DatasetStatusProcessing, } conv := &stubConverter{toCOGErr: errors.New("gdal failed")} - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, noopRowParser, conv) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, noopRowParser, conv) if err := svc.ConvertToCOG(context.Background(), id); err != nil { t.Fatalf("conversion failure should be recorded, not returned: %v", err) @@ -751,7 +772,7 @@ func TestDatasetService_Parse(t *testing.T) { } cols := []domain.AttributeColumn{{Name: "като"}, {Name: "F_2023"}} parser := ColumnParser(func(string, []byte) ([]domain.AttributeColumn, error) { return cols, nil }) - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, parser, noopRowParser, &stubConverter{}) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, parser, noopRowParser, &stubConverter{}) if err := svc.Parse(context.Background(), id); err != nil { t.Fatalf("unexpected error: %v", err) @@ -775,7 +796,7 @@ func TestDatasetService_Parse_RecordsFailure(t *testing.T) { parser := ColumnParser(func(string, []byte) ([]domain.AttributeColumn, error) { return nil, errors.New("corrupt archive") }) - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, parser, noopRowParser, &stubConverter{}) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, parser, noopRowParser, &stubConverter{}) if err := svc.Parse(context.Background(), id); err != nil { t.Fatalf("parse failure should be recorded, not returned: %v", err) @@ -886,7 +907,7 @@ func TestDatasetService_SaveMapping(t *testing.T) { repo := newStubDatasetRepo() repo.store[id] = base enq := &stubEnqueuer{} - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{}) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{}) got, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: validYears}) if err != nil { @@ -992,7 +1013,7 @@ func TestDatasetService_Extract(t *testing.T) { geoms := []json.RawMessage{json.RawMessage(`{"type":"Polygon","coordinates":[[[76,43],[77,43],[77,44],[76,43]]]}`)} geom := []byte(`{"type":"GeometryCollection","geometries":[` + string(geoms[0]) + `]}`) conv := &stubConverter{vectorFeatures: geoms} - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, conv) if err := svc.Extract(context.Background(), id); err != nil { t.Fatalf("unexpected error: %v", err) @@ -1427,7 +1448,7 @@ func TestDatasetService_Reprocess_ClearsStaleGeometryAndGeoJSON(t *testing.T) { GeoJSON: json.RawMessage(`{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"rows":[]}}]}`), } enq := &stubEnqueuer{} - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{}) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{}) if _, err := svc.Reprocess(context.Background(), id); err != nil { t.Fatalf("unexpected error: %v", err) @@ -1454,7 +1475,7 @@ func TestDatasetService_ReprocessAll_FiltersByFileType(t *testing.T) { Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[1,1],[2,2],[3,1],[1,1]]]}`), } enq := &stubEnqueuer{} - svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{}) + svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{}) vectorType := domain.FileTypeVector enqueued, failures, err := svc.ReprocessAll(context.Background(), &vectorType) @@ -1738,9 +1759,11 @@ func TestDatasetService_Patch_NotFound(t *testing.T) { func TestDatasetService_Upload_StoresPeriodAndGroup(t *testing.T) { repo := newStubDatasetRepo() - svc := newDatasetService(repo, &stubStore{}, true) + groups := newStubGroupRepo() + svc := newDatasetServiceWithGroups(repo, newStubStore(), true, groups) gid := uuid.New() + groups.store[gid] = domain.DatasetGroup{ID: gid, Code: "pop", Name: "Population"} start, end := "2020-01-01", "2020-12-31" in := validUpload() in.DatasetGroupID = &gid @@ -1772,3 +1795,134 @@ func TestDatasetService_Upload_RejectsReversedPeriod(t *testing.T) { t.Fatalf("nothing should be uploaded on validation failure, got %v", store.put) } } + +func TestDatasetService_DownloadSLD_FallsBackToGroup(t *testing.T) { + ctx := context.Background() + repo := newStubDatasetRepo() + store := newStubStore() + groups := newStubGroupRepo() + svc := newDatasetServiceWithGroups(repo, store, true, groups) + + groupSLD := "groups/style.sld" + filename, contentType := "group.sld", "application/xml" + group := domain.DatasetGroup{ + ID: uuid.New(), Code: "pop", Name: "Population", + SldStorageKey: &groupSLD, SldFilename: &filename, SldContentType: &contentType, + } + groups.store[group.ID] = group + store.objects[groupSLD] = []byte("") + + id := uuid.New() + repo.store[id] = domain.Dataset{ID: id, DatasetGroupID: &group.ID} + + dataset, obj, err := svc.DownloadSLD(ctx, id) + if err != nil { + t.Fatal(err) + } + defer obj.Close() + + if dataset.SldStorageKey == nil || *dataset.SldStorageKey != groupSLD { + t.Fatalf("want group sld key %s, got %v", groupSLD, dataset.SldStorageKey) + } + if dataset.SldFilename == nil || *dataset.SldFilename != filename { + t.Fatalf("want group sld filename %s, got %v", filename, dataset.SldFilename) + } + body, err := io.ReadAll(obj) + if err != nil { + t.Fatal(err) + } + if string(body) != "" { + t.Fatalf("want the group's sld content, got %q", body) + } +} + +func TestDatasetService_DownloadSLD_NoFallbackWhenGroupHasNoSLD(t *testing.T) { + repo := newStubDatasetRepo() + groups := newStubGroupRepo() + svc := newDatasetServiceWithGroups(repo, newStubStore(), true, groups) + + group := domain.DatasetGroup{ID: uuid.New(), Code: "pop", Name: "Population"} + groups.store[group.ID] = group + id := uuid.New() + repo.store[id] = domain.Dataset{ID: id, DatasetGroupID: &group.ID} + + if _, _, err := svc.DownloadSLD(context.Background(), id); !errors.Is(err, ErrSLDNotFound) { + t.Fatalf("want ErrSLDNotFound, got %v", err) + } +} + +func TestDatasetService_Upload_RejectsUnknownGroup(t *testing.T) { + svc := newDatasetService(newStubDatasetRepo(), newStubStore(), true) + in := validUpload() + missing := uuid.New() + in.DatasetGroupID = &missing + + if _, err := svc.Upload(context.Background(), in); !errors.Is(err, domain.ErrValidation) { + t.Fatalf("want ErrValidation, got %v", err) + } +} + +func TestDatasetService_CreateRasterFromFile(t *testing.T) { + ctx := context.Background() + repo := newStubDatasetRepo() + store := newStubStore() + groups := newStubGroupRepo() + jobs := &stubEnqueuer{} + svc := NewDatasetService(repo, store, stubCategoryReader{exists: true}, groups, jobs, noopParser, noopRowParser, &stubConverter{}) + + group := domain.DatasetGroup{ID: uuid.New(), Code: "lp-z", Name: "Landscape Z"} + groups.store[group.ID] = group + + // A minimal little-endian TIFF header is enough for content sniffing. + path := filepath.Join(t.TempDir(), "lp-z-integrated.tif") + if err := os.WriteFile(path, []byte("II*\x00 raster payload"), 0o600); err != nil { + t.Fatal(err) + } + end := "2024-12-31" + + dataset, err := svc.CreateRasterFromFile(ctx, CreateRasterInput{ + CategoryID: uuid.New(), + DatasetGroupID: &group.ID, + Name: "Landscape Z integrated", + Filename: "lp-z-integrated.tif", + Path: path, + EndDate: &end, + Automated: true, + }) + if err != nil { + t.Fatal(err) + } + if dataset.FileType != domain.FileTypeRaster { + t.Fatalf("want raster file type, got %q", dataset.FileType) + } + if dataset.DatasetGroupID == nil || *dataset.DatasetGroupID != group.ID { + t.Fatalf("want group %s, got %v", group.ID, dataset.DatasetGroupID) + } + if dataset.EndDate == nil || *dataset.EndDate != end { + t.Fatalf("want end date %s, got %v", end, dataset.EndDate) + } + if !dataset.Automated { + t.Fatal("want automated dataset") + } + if dataset.SizeBytes != int64(len("II*\x00 raster payload")) { + t.Fatalf("want the file size, got %d", dataset.SizeBytes) + } + if len(store.put) != 1 { + t.Fatalf("want the raster stored once, got %v", store.put) + } + if len(jobs.converted) != 1 || jobs.converted[0] != dataset.ID { + t.Fatalf("want a convert job for %s, got %v", dataset.ID, jobs.converted) + } +} + +func TestDatasetService_CreateRasterFromFile_MissingFile(t *testing.T) { + svc := newDatasetService(newStubDatasetRepo(), newStubStore(), true) + _, err := svc.CreateRasterFromFile(context.Background(), CreateRasterInput{ + CategoryID: uuid.New(), + Filename: "missing.tif", + Path: filepath.Join(t.TempDir(), "missing.tif"), + }) + if err == nil { + t.Fatal("want an error for a missing file") + } +} diff --git a/internal/transport/http/dataset_group_handler.go b/internal/transport/http/dataset_group_handler.go new file mode 100644 index 0000000..13970de --- /dev/null +++ b/internal/transport/http/dataset_group_handler.go @@ -0,0 +1,236 @@ +package http + +import ( + "io" + "net/http" + "strings" + + "gis/internal/domain" + "gis/internal/service" + "gis/pkg/httputil" + + "github.com/go-chi/chi/v5" + "github.com/go-playground/validator/v10" +) + +// DatasetGroupHandler serves the /dataset-groups routes. +type DatasetGroupHandler struct { + svc *service.DatasetGroupService + validate *validator.Validate +} + +// NewDatasetGroupHandler returns a DatasetGroupHandler. +func NewDatasetGroupHandler(svc *service.DatasetGroupService, validate *validator.Validate) *DatasetGroupHandler { + return &DatasetGroupHandler{svc: svc, validate: validate} +} + +// Register mounts the dataset-group routes on r. +func (h *DatasetGroupHandler) Register(r chi.Router) { + r.Get("/", h.list) + r.Post("/", h.create) + r.Get("/{id}", h.get) + r.Put("/{id}", h.update) + r.Delete("/{id}", h.delete) + r.Get("/{id}/latest", h.latest) + r.Post("/{id}/sld", h.uploadSLD) + r.Get("/{id}/sld", h.downloadSLD) +} + +type datasetGroupRequest struct { + Code string `json:"code" validate:"required,max=255,slug"` + Name string `json:"name" validate:"required,max=255"` + Description string `json:"description" validate:"max=2000"` +} + +func (r datasetGroupRequest) toInput() service.DatasetGroupInput { + return service.DatasetGroupInput{Code: r.Code, Name: r.Name, Description: r.Description} +} + +func (h *DatasetGroupHandler) create(w http.ResponseWriter, r *http.Request) { + req, err := httputil.DecodeJSON[datasetGroupRequest](w, r) + if err != nil { + httputil.WriteError(w, http.StatusBadRequest, "invalid request body") + return + } + if err := h.validate.Struct(req); err != nil { + httputil.WriteValidationErrors(w, err) + return + } + + group, err := h.svc.Create(r.Context(), req.toInput()) + if err != nil { + respondDomainError(w, err) + return + } + httputil.WriteJSON(w, http.StatusCreated, group) +} + +func (h *DatasetGroupHandler) list(w http.ResponseWriter, r *http.Request) { + var code *string + if v := strings.TrimSpace(r.URL.Query().Get("code")); v != "" { + code = &v + } + + page, ok := parsePositiveIntQuery(w, r, "page", 1) + if !ok { + return + } + pageSize, ok := parsePositiveIntQuery(w, r, "page_size", service.DefaultPageSize) + if !ok { + return + } + + result, err := h.svc.List(r.Context(), code, page, pageSize) + if err != nil { + respondDomainError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, newPaginated(result.Items, result.Page, result.PageSize, result.Total)) +} + +func (h *DatasetGroupHandler) get(w http.ResponseWriter, r *http.Request) { + id, ok := parseUUIDParam(w, r, "id") + if !ok { + return + } + group, err := h.svc.Get(r.Context(), id) + if err != nil { + respondDomainError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, group) +} + +func (h *DatasetGroupHandler) update(w http.ResponseWriter, r *http.Request) { + id, ok := parseUUIDParam(w, r, "id") + if !ok { + return + } + req, err := httputil.DecodeJSON[datasetGroupRequest](w, r) + if err != nil { + httputil.WriteError(w, http.StatusBadRequest, "invalid request body") + return + } + if err := h.validate.Struct(req); err != nil { + httputil.WriteValidationErrors(w, err) + return + } + + group, err := h.svc.Update(r.Context(), id, req.toInput()) + if err != nil { + respondDomainError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, group) +} + +// delete removes a group. A group still referenced by datasets yields 409. +func (h *DatasetGroupHandler) delete(w http.ResponseWriter, r *http.Request) { + id, ok := parseUUIDParam(w, r, "id") + if !ok { + return + } + if err := h.svc.Delete(r.Context(), id); err != nil { + respondDomainError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// latest returns the freshest dataset of the group (greatest end_date, NULLs +// last, then newest), optionally narrowed by ?status= and ?file_type=. +func (h *DatasetGroupHandler) latest(w http.ResponseWriter, r *http.Request) { + id, ok := parseUUIDParam(w, r, "id") + if !ok { + return + } + + var status *string + if v := strings.TrimSpace(r.URL.Query().Get("status")); v != "" { + if !domain.ValidDatasetStatus(v) { + httputil.WriteError(w, http.StatusBadRequest, "invalid status") + return + } + status = &v + } + + var fileType *domain.FileType + if v := strings.TrimSpace(r.URL.Query().Get("file_type")); v != "" { + ft := domain.FileType(v) + if !ft.Valid() { + httputil.WriteError(w, http.StatusBadRequest, "file_type must be one of: vector_with_kato, vector, raster") + return + } + fileType = &ft + } + + dataset, err := h.svc.Latest(r.Context(), id, status, fileType) + if err != nil { + respondDomainError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, dataset) +} + +func (h *DatasetGroupHandler) uploadSLD(w http.ResponseWriter, r *http.Request) { + id, ok := parseUUIDParam(w, r, "id") + if !ok { + return + } + if err := r.ParseMultipartForm(maxUploadBytes); err != nil { + httputil.WriteError(w, http.StatusBadRequest, "request must be multipart/form-data") + return + } + + file, header, err := r.FormFile("file") + if err != nil { + httputil.WriteError(w, http.StatusBadRequest, "file is required") + return + } + defer file.Close() + + contentType := header.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/xml" + } + + group, err := h.svc.UploadSLD(r.Context(), id, service.UploadSLDInput{ + Filename: header.Filename, + ContentType: contentType, + Size: header.Size, + Reader: file, + }) + if err != nil { + respondDomainError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, group) +} + +func (h *DatasetGroupHandler) downloadSLD(w http.ResponseWriter, r *http.Request) { + id, ok := parseUUIDParam(w, r, "id") + if !ok { + return + } + group, obj, err := h.svc.DownloadSLD(r.Context(), id) + if err != nil { + respondDomainError(w, err) + return + } + defer obj.Close() + + contentType := "application/xml" + if group.SldContentType != nil && *group.SldContentType != "" { + contentType = *group.SldContentType + } + filename := "style.sld" + if group.SldFilename != nil && *group.SldFilename != "" { + filename = *group.SldFilename + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`) + if _, err := io.Copy(w, obj); err != nil { + // Headers are already sent; nothing useful to return to the client. + return + } +} diff --git a/internal/transport/http/router.go b/internal/transport/http/router.go index ce01181..713a590 100644 --- a/internal/transport/http/router.go +++ b/internal/transport/http/router.go @@ -10,11 +10,13 @@ import ( // RouterDeps holds the handlers and dependencies the router mounts. type RouterDeps struct { - Logger *slog.Logger - Health *HealthHandler - Categories *CategoryHandler - Datasets *DatasetHandler - OpenAPISpec []byte + Logger *slog.Logger + Health *HealthHandler + Categories *CategoryHandler + Datasets *DatasetHandler + // DatasetGroups serves the /dataset-groups routes. + DatasetGroups *DatasetGroupHandler + OpenAPISpec []byte } // NewRouter builds the application's HTTP handler with middleware and routes. @@ -37,6 +39,7 @@ func NewRouter(deps RouterDeps) http.Handler { // Domain routes. Add future feature routers here. r.Route("/categories", deps.Categories.Register) r.Route("/datasets", deps.Datasets.Register) + r.Route("/dataset-groups", deps.DatasetGroups.Register) return r }