feat(dataset-groups): CRUD, latest-in-group, group SLD with dataset fallback, CreateRasterFromFile

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
2026-09-05 13:29:01 +05:00
parent 32eba016b3
commit bc524f3d98
11 changed files with 1366 additions and 34 deletions

View File

@@ -86,6 +86,12 @@ type categoryReader interface {
GetByCode(ctx context.Context, code string) (domain.Category, error)
}
// groupReader lets the dataset service verify a dataset group exists before a
// dataset is filed under it, and resolve its shared SLD as a fallback.
type groupReader interface {
GetByID(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, error)
}
// JobEnqueuer schedules asynchronous dataset jobs.
type JobEnqueuer interface {
EnqueueParse(ctx context.Context, datasetID uuid.UUID) error
@@ -138,6 +144,7 @@ type DatasetService struct {
repo DatasetRepository
store ObjectStore
categories categoryReader
groups groupReader
jobs JobEnqueuer
parseColumns ColumnParser
parseRows RowParser
@@ -145,12 +152,14 @@ type DatasetService struct {
}
// NewDatasetService wires the dataset repository, object store, category reader
// (for parent validation), the job enqueuer, the column/row parsers, and the
// raster converter.
// (for parent validation), the dataset-group reader (for group validation and
// the SLD fallback), the job enqueuer, the column/row parsers, and the raster
// converter.
func NewDatasetService(
repo DatasetRepository,
store ObjectStore,
categories categoryReader,
groups groupReader,
jobs JobEnqueuer,
parseColumns ColumnParser,
parseRows RowParser,
@@ -160,6 +169,7 @@ func NewDatasetService(
repo: repo,
store: store,
categories: categories,
groups: groups,
jobs: jobs,
parseColumns: parseColumns,
parseRows: parseRows,
@@ -196,6 +206,9 @@ func (s *DatasetService) Upload(ctx context.Context, in UploadInput) (domain.Dat
if err := s.ensureCategoryExists(ctx, in.CategoryID); err != nil {
return domain.Dataset{}, err
}
if err := s.ensureGroupExists(ctx, in.DatasetGroupID); err != nil {
return domain.Dataset{}, err
}
if err := domain.ValidateDatePeriod(in.StartDate, in.EndDate); err != nil {
return domain.Dataset{}, err
}
@@ -311,7 +324,23 @@ func (s *DatasetService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain.
return domain.Dataset{}, nil, err
}
if dataset.SldStorageKey == nil {
return domain.Dataset{}, nil, ErrSLDNotFound
// Fall back to the dataset group's shared style, if the group has one.
if dataset.DatasetGroupID == nil {
return domain.Dataset{}, nil, ErrSLDNotFound
}
group, err := s.groups.GetByID(ctx, *dataset.DatasetGroupID)
if err != nil {
if errors.Is(err, domain.ErrNotFound) {
return domain.Dataset{}, nil, ErrSLDNotFound
}
return domain.Dataset{}, nil, err
}
if group.SldStorageKey == nil {
return domain.Dataset{}, nil, ErrSLDNotFound
}
dataset.SldStorageKey = group.SldStorageKey
dataset.SldFilename = group.SldFilename
dataset.SldContentType = group.SldContentType
}
obj, err := s.store.Get(ctx, *dataset.SldStorageKey)
if err != nil {
@@ -320,6 +349,48 @@ func (s *DatasetService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain.
return dataset, obj, nil
}
// CreateRasterInput describes a raster file already on local disk that should
// become a dataset (used by the Landscape Program to publish results).
type CreateRasterInput struct {
CategoryID uuid.UUID
DatasetGroupID *uuid.UUID
Name string
Filename string // stored filename, e.g. "lp-z-integrated.tif"
Path string // local path to read from
StartDate *string
EndDate *string
Meta json.RawMessage
Automated bool
}
// CreateRasterFromFile stores a local GeoTIFF as a new raster dataset and
// enqueues the standard COG conversion, exactly like an uploaded raster.
func (s *DatasetService) CreateRasterFromFile(ctx context.Context, in CreateRasterInput) (domain.Dataset, error) {
f, err := os.Open(in.Path)
if err != nil {
return domain.Dataset{}, err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return domain.Dataset{}, err
}
return s.Upload(ctx, UploadInput{
CategoryID: in.CategoryID,
DatasetGroupID: in.DatasetGroupID,
Name: in.Name,
Meta: in.Meta,
Automated: in.Automated,
Filename: in.Filename,
FileType: domain.FileTypeRaster,
ContentType: "image/tiff",
Size: info.Size(),
Reader: f,
StartDate: in.StartDate,
EndDate: in.EndDate,
})
}
// deriveSLDKey places a dataset's SLD style file alongside the original under
// an sld/ prefix, named after the uploaded SLD file.
func deriveSLDKey(storageKey, filename string) string {
@@ -1099,6 +1170,9 @@ func (s *DatasetService) Update(ctx context.Context, id uuid.UUID, in DatasetUpd
if err := s.ensureCategoryExists(ctx, in.CategoryID); err != nil {
return domain.Dataset{}, err
}
if err := s.ensureGroupExists(ctx, in.DatasetGroupID); err != nil {
return domain.Dataset{}, err
}
if err := domain.ValidateDatePeriod(in.StartDate, in.EndDate); err != nil {
return domain.Dataset{}, err
}
@@ -1163,6 +1237,9 @@ func (s *DatasetService) Patch(ctx context.Context, id uuid.UUID, in DatasetPatc
groupID := current.DatasetGroupID
if in.DatasetGroupIDSet {
groupID = in.DatasetGroupID
if err := s.ensureGroupExists(ctx, groupID); err != nil {
return domain.Dataset{}, err
}
}
startDate := current.StartDate
if in.StartDateSet {
@@ -1199,6 +1276,21 @@ func (s *DatasetService) ensureCategoryExists(ctx context.Context, categoryID uu
return nil
}
// ensureGroupExists validates that a non-nil dataset group id references an
// existing group. A nil id (no group) is always valid.
func (s *DatasetService) ensureGroupExists(ctx context.Context, id *uuid.UUID) error {
if id == nil {
return nil
}
if _, err := s.groups.GetByID(ctx, *id); err != nil {
if errors.Is(err, domain.ErrNotFound) {
return fmt.Errorf("%w: dataset group %s not found", domain.ErrValidation, id)
}
return err
}
return nil
}
// Status-wait bounds and polling cadence for long polling.
const (
DefaultStatusWait = 25 * time.Second

View File

@@ -0,0 +1,185 @@
package service
import (
"bytes"
"context"
"fmt"
"io"
"path"
"path/filepath"
"strings"
"gis/internal/domain"
"github.com/google/uuid"
)
// DatasetGroupRepository is the persistence behaviour DatasetGroupService needs.
type DatasetGroupRepository interface {
Create(ctx context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error)
GetByID(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, error)
GetByCode(ctx context.Context, code string) (domain.DatasetGroup, error)
List(ctx context.Context, code *string, limit, offset int) ([]domain.DatasetGroup, error)
Count(ctx context.Context, code *string) (int, error)
Update(ctx context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error)
Delete(ctx context.Context, id uuid.UUID) error
SetSLD(ctx context.Context, id uuid.UUID, key, filename, contentType string) (domain.DatasetGroup, error)
}
// DatasetGroupInput carries the mutable fields of a group.
type DatasetGroupInput struct {
Code string
Name string
Description string
}
// DatasetGroupPage is a page of groups.
type DatasetGroupPage struct {
Items []domain.DatasetGroup
Page int
PageSize int
Total int
}
// DatasetGroupService implements dataset-group business rules.
type DatasetGroupService struct {
repo DatasetGroupRepository
datasets DatasetRepository
store ObjectStore
}
// NewDatasetGroupService wires the group repository, the dataset repository
// (for the "latest" lookup) and the object store (for the group SLD).
func NewDatasetGroupService(repo DatasetGroupRepository, datasets DatasetRepository, store ObjectStore) *DatasetGroupService {
return &DatasetGroupService{repo: repo, datasets: datasets, store: store}
}
// errInvalidGroupCode is the validation error returned for a non-slug code.
func errInvalidGroupCode() error {
return fmt.Errorf("%w: code must be a slug (lowercase latin letters, digits, and dashes)", domain.ErrValidation)
}
// Create persists a new group after validating its code.
func (s *DatasetGroupService) Create(ctx context.Context, in DatasetGroupInput) (domain.DatasetGroup, error) {
if !domain.ValidSlug(in.Code) {
return domain.DatasetGroup{}, errInvalidGroupCode()
}
return s.repo.Create(ctx, domain.DatasetGroup{Code: in.Code, Name: in.Name, Description: in.Description})
}
// Get returns one group by id.
func (s *DatasetGroupService) Get(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, error) {
return s.repo.GetByID(ctx, id)
}
// List returns a page of groups; page/pageSize are clamped like datasets.
func (s *DatasetGroupService) List(ctx context.Context, code *string, page, pageSize int) (DatasetGroupPage, error) {
if page < 1 {
page = 1
}
if pageSize < 1 {
pageSize = DefaultPageSize
}
if pageSize > MaxPageSize {
pageSize = MaxPageSize
}
items, err := s.repo.List(ctx, code, pageSize, (page-1)*pageSize)
if err != nil {
return DatasetGroupPage{}, err
}
total, err := s.repo.Count(ctx, code)
if err != nil {
return DatasetGroupPage{}, err
}
return DatasetGroupPage{Items: items, Page: page, PageSize: pageSize, Total: total}, nil
}
// Update replaces a group's editable fields.
func (s *DatasetGroupService) Update(ctx context.Context, id uuid.UUID, in DatasetGroupInput) (domain.DatasetGroup, error) {
if !domain.ValidSlug(in.Code) {
return domain.DatasetGroup{}, errInvalidGroupCode()
}
return s.repo.Update(ctx, domain.DatasetGroup{ID: id, Code: in.Code, Name: in.Name, Description: in.Description})
}
// Delete removes a group (409 via ErrConflict while datasets reference it) and
// then best-effort removes its SLD object.
func (s *DatasetGroupService) Delete(ctx context.Context, id uuid.UUID) error {
g, err := s.repo.GetByID(ctx, id)
if err != nil {
return err
}
if err := s.repo.Delete(ctx, id); err != nil {
return err
}
if g.SldStorageKey != nil {
_ = s.store.Remove(ctx, *g.SldStorageKey) // best-effort: the row is gone
}
return nil
}
// Latest returns the freshest dataset of the group (greatest end_date, NULLs
// last, then newest), optionally restricted by status and file type.
func (s *DatasetGroupService) Latest(ctx context.Context, id uuid.UUID, status *string, fileType *domain.FileType) (domain.Dataset, error) {
if _, err := s.repo.GetByID(ctx, id); err != nil {
return domain.Dataset{}, err
}
return s.datasets.LatestInGroup(ctx, id, status, fileType)
}
// UploadSLD validates and stores the group's shared SLD, replacing any previous one.
func (s *DatasetGroupService) UploadSLD(ctx context.Context, id uuid.UUID, in UploadSLDInput) (domain.DatasetGroup, error) {
g, err := s.repo.GetByID(ctx, id)
if err != nil {
return domain.DatasetGroup{}, err
}
ext := strings.ToLower(filepath.Ext(in.Filename))
if !domain.ExtensionAllowedForSLD(ext) {
return domain.DatasetGroup{}, fmt.Errorf("%w: sld file must have one of these extensions: %s",
domain.ErrValidation, strings.Join(domain.AllowedSLDExtensions, ", "))
}
head := make([]byte, 512)
n, err := io.ReadFull(in.Reader, head)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return domain.DatasetGroup{}, fmt.Errorf("read sld upload: %w", err)
}
head = head[:n]
if err := domain.ValidateFileContent(ext, head); err != nil {
return domain.DatasetGroup{}, fmt.Errorf("%w: %s", domain.ErrValidation, err)
}
content := io.MultiReader(bytes.NewReader(head), in.Reader)
key := path.Join("groups", g.ID.String(), "sld", in.Filename)
if err := s.store.Put(ctx, key, content, in.Size, in.ContentType); err != nil {
return domain.DatasetGroup{}, err
}
updated, err := s.repo.SetSLD(ctx, id, key, in.Filename, in.ContentType)
if err != nil {
// Compensate: the row was not updated, so the object would be orphaned.
_ = s.store.Remove(ctx, key)
return domain.DatasetGroup{}, err
}
if g.SldStorageKey != nil && *g.SldStorageKey != key {
_ = s.store.Remove(ctx, *g.SldStorageKey) // best-effort: drop the replaced file
}
return updated, nil
}
// DownloadSLD returns the group and a reader for its SLD; ErrSLDNotFound when
// none has been uploaded. The caller must close the reader.
func (s *DatasetGroupService) DownloadSLD(ctx context.Context, id uuid.UUID) (domain.DatasetGroup, io.ReadCloser, error) {
g, err := s.repo.GetByID(ctx, id)
if err != nil {
return domain.DatasetGroup{}, nil, err
}
if g.SldStorageKey == nil {
return domain.DatasetGroup{}, nil, ErrSLDNotFound
}
obj, err := s.store.Get(ctx, *g.SldStorageKey)
if err != nil {
return domain.DatasetGroup{}, nil, err
}
return g, obj, nil
}

View File

@@ -0,0 +1,257 @@
package service
import (
"context"
"errors"
"strings"
"testing"
"time"
"gis/internal/domain"
"github.com/google/uuid"
)
// stubGroupRepo is an in-memory DatasetGroupRepository for tests. It also
// satisfies the dataset service's groupReader dependency.
type stubGroupRepo struct {
store map[uuid.UUID]domain.DatasetGroup
}
func newStubGroupRepo() *stubGroupRepo {
return &stubGroupRepo{store: map[uuid.UUID]domain.DatasetGroup{}}
}
func (r *stubGroupRepo) Create(_ context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) {
for _, e := range r.store {
if e.Code == g.Code {
return domain.DatasetGroup{}, domain.ErrConflict
}
}
g.ID = uuid.New()
r.store[g.ID] = g
return g, nil
}
func (r *stubGroupRepo) GetByID(_ context.Context, id uuid.UUID) (domain.DatasetGroup, error) {
g, ok := r.store[id]
if !ok {
return domain.DatasetGroup{}, domain.ErrNotFound
}
return g, nil
}
func (r *stubGroupRepo) GetByCode(_ context.Context, code string) (domain.DatasetGroup, error) {
for _, g := range r.store {
if g.Code == code {
return g, nil
}
}
return domain.DatasetGroup{}, domain.ErrNotFound
}
func (r *stubGroupRepo) List(_ context.Context, code *string, _, _ int) ([]domain.DatasetGroup, error) {
out := []domain.DatasetGroup{}
for _, g := range r.store {
if code == nil || g.Code == *code {
out = append(out, g)
}
}
return out, nil
}
func (r *stubGroupRepo) Count(_ context.Context, code *string) (int, error) {
l, _ := r.List(context.Background(), code, 0, 0)
return len(l), nil
}
func (r *stubGroupRepo) Update(_ context.Context, g domain.DatasetGroup) (domain.DatasetGroup, error) {
if _, ok := r.store[g.ID]; !ok {
return domain.DatasetGroup{}, domain.ErrNotFound
}
r.store[g.ID] = g
return g, nil
}
func (r *stubGroupRepo) Delete(_ context.Context, id uuid.UUID) error {
if _, ok := r.store[id]; !ok {
return domain.ErrNotFound
}
delete(r.store, id)
return nil
}
func (r *stubGroupRepo) SetSLD(_ context.Context, id uuid.UUID, key, filename, contentType string) (domain.DatasetGroup, error) {
g, ok := r.store[id]
if !ok {
return domain.DatasetGroup{}, domain.ErrNotFound
}
g.SldStorageKey, g.SldFilename, g.SldContentType = &key, &filename, &contentType
r.store[id] = g
return g, nil
}
func TestDatasetGroupService_Create_RequiresSlug(t *testing.T) {
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore())
_, err := svc.Create(context.Background(), DatasetGroupInput{Code: "Bad Code", Name: "x"})
if !errors.Is(err, domain.ErrValidation) {
t.Fatalf("want ErrValidation, got %v", err)
}
}
func TestDatasetGroupService_Latest_PrefersGreatestEndDateThenCreatedAt(t *testing.T) {
groups := newStubGroupRepo()
datasets := newStubDatasetRepo()
svc := NewDatasetGroupService(groups, datasets, newStubStore())
g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"})
if err != nil {
t.Fatal(err)
}
s := func(v string) *string { return &v }
base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
old := domain.Dataset{ID: uuid.New(), DatasetGroupID: &g.ID, EndDate: s("2019-12-31"), Status: domain.DatasetStatusReady, CreatedAt: base}
fresh := domain.Dataset{ID: uuid.New(), DatasetGroupID: &g.ID, EndDate: s("2020-12-31"), Status: domain.DatasetStatusReady, CreatedAt: base.Add(time.Hour)}
undated := domain.Dataset{ID: uuid.New(), DatasetGroupID: &g.ID, Status: domain.DatasetStatusReady, CreatedAt: base.Add(2 * time.Hour)}
for _, d := range []domain.Dataset{old, fresh, undated} {
datasets.store[d.ID] = d
}
got, err := svc.Latest(context.Background(), g.ID, nil, nil)
if err != nil {
t.Fatal(err)
}
if got.ID != fresh.ID {
t.Fatalf("want %s, got %s", fresh.ID, got.ID)
}
}
func TestDatasetGroupService_Latest_NotFoundWhenEmpty(t *testing.T) {
groups := newStubGroupRepo()
svc := NewDatasetGroupService(groups, newStubDatasetRepo(), newStubStore())
g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"})
if err != nil {
t.Fatal(err)
}
if _, err := svc.Latest(context.Background(), g.ID, nil, nil); !errors.Is(err, domain.ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestDatasetGroupService_Latest_UnknownGroupIsNotFound(t *testing.T) {
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore())
if _, err := svc.Latest(context.Background(), uuid.New(), nil, nil); !errors.Is(err, domain.ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestDatasetGroupService_UploadSLD_RejectsBadExtension(t *testing.T) {
groups := newStubGroupRepo()
store := newStubStore()
svc := NewDatasetGroupService(groups, newStubDatasetRepo(), store)
g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"})
if err != nil {
t.Fatal(err)
}
_, err = svc.UploadSLD(context.Background(), g.ID, UploadSLDInput{Filename: "style.txt", ContentType: "text/plain", Size: 3, Reader: strings.NewReader("<x>")})
if !errors.Is(err, domain.ErrValidation) {
t.Fatalf("want ErrValidation, got %v", err)
}
if len(store.put) != 0 {
t.Fatalf("nothing should be stored on validation failure, got %v", store.put)
}
}
func TestDatasetGroupService_UploadSLD_StoresAndReplaces(t *testing.T) {
ctx := context.Background()
groups := newStubGroupRepo()
store := newStubStore()
svc := NewDatasetGroupService(groups, newStubDatasetRepo(), store)
g, err := svc.Create(ctx, DatasetGroupInput{Code: "pop", Name: "Population"})
if err != nil {
t.Fatal(err)
}
updated, err := svc.UploadSLD(ctx, g.ID, UploadSLDInput{
Filename: "style.sld", ContentType: "application/xml", Size: 3, Reader: strings.NewReader("<x>"),
})
if err != nil {
t.Fatal(err)
}
wantKey := "groups/" + g.ID.String() + "/sld/style.sld"
if updated.SldStorageKey == nil || *updated.SldStorageKey != wantKey {
t.Fatalf("want key %s, got %v", wantKey, updated.SldStorageKey)
}
// A second upload under a different name replaces the previous object.
if _, err := svc.UploadSLD(ctx, g.ID, UploadSLDInput{
Filename: "other.sld", ContentType: "application/xml", Size: 3, Reader: strings.NewReader("<y>"),
}); err != nil {
t.Fatal(err)
}
if len(store.removed) != 1 || store.removed[0] != wantKey {
t.Fatalf("want previous sld removed, got %v", store.removed)
}
}
func TestDatasetGroupService_DownloadSLD_NotFoundWhenAbsent(t *testing.T) {
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore())
g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"})
if err != nil {
t.Fatal(err)
}
if _, _, err := svc.DownloadSLD(context.Background(), g.ID); !errors.Is(err, ErrSLDNotFound) {
t.Fatalf("want ErrSLDNotFound, got %v", err)
}
}
func TestDatasetGroupService_Delete_RemovesSLDObject(t *testing.T) {
ctx := context.Background()
store := newStubStore()
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), store)
g, err := svc.Create(ctx, DatasetGroupInput{Code: "pop", Name: "Population"})
if err != nil {
t.Fatal(err)
}
if _, err := svc.UploadSLD(ctx, g.ID, UploadSLDInput{
Filename: "style.sld", ContentType: "application/xml", Size: 3, Reader: strings.NewReader("<x>"),
}); err != nil {
t.Fatal(err)
}
if err := svc.Delete(ctx, g.ID); err != nil {
t.Fatal(err)
}
wantKey := "groups/" + g.ID.String() + "/sld/style.sld"
if len(store.removed) != 1 || store.removed[0] != wantKey {
t.Fatalf("want sld object removed, got %v", store.removed)
}
if _, err := svc.Get(ctx, g.ID); !errors.Is(err, domain.ErrNotFound) {
t.Fatalf("want ErrNotFound after delete, got %v", err)
}
}
func TestDatasetGroupService_List_ClampsPageSize(t *testing.T) {
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore())
if _, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"}); err != nil {
t.Fatal(err)
}
page, err := svc.List(context.Background(), nil, 0, 10_000)
if err != nil {
t.Fatal(err)
}
if page.Page != 1 || page.PageSize != MaxPageSize {
t.Fatalf("want page 1 and page size %d, got %d/%d", MaxPageSize, page.Page, page.PageSize)
}
if page.Total != 1 {
t.Fatalf("want total 1, got %d", page.Total)
}
}
func TestDatasetGroupService_Update_RequiresSlug(t *testing.T) {
svc := NewDatasetGroupService(newStubGroupRepo(), newStubDatasetRepo(), newStubStore())
g, err := svc.Create(context.Background(), DatasetGroupInput{Code: "pop", Name: "Population"})
if err != nil {
t.Fatal(err)
}
if _, err := svc.Update(context.Background(), g.ID, DatasetGroupInput{Code: "Bad Code", Name: "x"}); !errors.Is(err, domain.ErrValidation) {
t.Fatalf("want ErrValidation, got %v", err)
}
}

View File

@@ -7,6 +7,7 @@ import (
"errors"
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"
@@ -358,22 +359,36 @@ var (
noopRowParser RowParser = func(string, []byte) ([]map[string]string, error) { return nil, nil }
)
// stubStore records object-store interactions.
// stubStore records object-store interactions and keeps stored bodies so tests
// can read them back.
type stubStore struct {
put []string
removed []string
putErr error
objects map[string][]byte
}
func (s *stubStore) Put(_ context.Context, key string, _ io.Reader, _ int64, _ string) error {
func newStubStore() *stubStore { return &stubStore{objects: map[string][]byte{}} }
func (s *stubStore) Put(_ context.Context, key string, r io.Reader, _ int64, _ string) error {
if s.putErr != nil {
return s.putErr
}
s.put = append(s.put, key)
if s.objects != nil {
body, err := io.ReadAll(r)
if err != nil {
return err
}
s.objects[key] = body
}
return nil
}
func (s *stubStore) Get(_ context.Context, _ string) (io.ReadCloser, error) {
func (s *stubStore) Get(_ context.Context, key string) (io.ReadCloser, error) {
if body, ok := s.objects[key]; ok {
return io.NopCloser(bytes.NewReader(body)), nil
}
return io.NopCloser(strings.NewReader("")), nil
}
@@ -414,7 +429,13 @@ func validUpload() UploadInput {
}
func newDatasetService(repo *stubDatasetRepo, store *stubStore, catExists bool) *DatasetService {
return NewDatasetService(repo, store, stubCategoryReader{exists: catExists}, &stubEnqueuer{}, noopParser, noopRowParser, &stubConverter{})
return newDatasetServiceWithGroups(repo, store, catExists, newStubGroupRepo())
}
// newDatasetServiceWithGroups builds a DatasetService over a caller-supplied
// group repository, for tests that need a pre-populated dataset group.
func newDatasetServiceWithGroups(repo *stubDatasetRepo, store *stubStore, catExists bool, groups *stubGroupRepo) *DatasetService {
return NewDatasetService(repo, store, stubCategoryReader{exists: catExists}, groups, &stubEnqueuer{}, noopParser, noopRowParser, &stubConverter{})
}
func TestDatasetService_Upload_Validation(t *testing.T) {
@@ -521,7 +542,7 @@ func TestDatasetService_Upload_CompensatesOnDBFailure(t *testing.T) {
func TestDatasetService_Upload_VectorWithKato_EnqueuesParse(t *testing.T) {
repo := newStubDatasetRepo()
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
in := validUpload()
in.FileType = domain.FileTypeVectorWithKato
@@ -542,7 +563,7 @@ func TestDatasetService_Upload_VectorWithKato_EnqueuesParse(t *testing.T) {
func TestDatasetService_Upload_Vector_EnqueuesProperties(t *testing.T) {
repo := newStubDatasetRepo()
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
got, err := svc.Upload(context.Background(), validUpload()) // plain vector .geojson
if err != nil {
@@ -573,7 +594,7 @@ func TestDatasetService_ExtractProperties(t *testing.T) {
json.RawMessage(`{"type":"Polygon","coordinates":[[[4,4],[5,5],[6,4],[4,4]]]}`),
}
conv := &stubConverter{vectorFeatures: geoms}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, conv)
if err := svc.ExtractProperties(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -625,7 +646,7 @@ func TestDatasetService_ExtractProperties_GeometryBestEffort(t *testing.T) {
conv := &stubConverter{vectorFeaturesFn: func(string) ([]json.RawMessage, error) {
return nil, errors.New("ogr2ogr boom")
}}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, conv)
if err := svc.ExtractProperties(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -653,7 +674,7 @@ func TestDatasetService_ExtractProperties_NoTable(t *testing.T) {
rp := RowParser(func(string, []byte) ([]map[string]string, error) {
return []map[string]string{{}, {}}, nil
})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, &stubConverter{})
if err := svc.ExtractProperties(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -670,7 +691,7 @@ func TestDatasetService_ExtractProperties_NoTable(t *testing.T) {
func TestDatasetService_Upload_Raster_EnqueuesConvert(t *testing.T) {
repo := newStubDatasetRepo()
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
in := validUpload()
in.FileType = domain.FileTypeRaster
@@ -700,7 +721,7 @@ func TestDatasetService_ConvertToCOG(t *testing.T) {
store := &stubStore{}
footprint := []byte(`{"type":"Polygon","coordinates":[[[70,50],[72,50],[72,52],[70,52],[70,50]]]}`)
conv := &stubConverter{footprint: footprint}
svc := NewDatasetService(repo, store, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, noopRowParser, conv)
svc := NewDatasetService(repo, store, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, noopRowParser, conv)
if err := svc.ConvertToCOG(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -732,7 +753,7 @@ func TestDatasetService_ConvertToCOG_RecordsFailure(t *testing.T) {
Status: domain.DatasetStatusProcessing,
}
conv := &stubConverter{toCOGErr: errors.New("gdal failed")}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, noopRowParser, conv)
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, noopRowParser, conv)
if err := svc.ConvertToCOG(context.Background(), id); err != nil {
t.Fatalf("conversion failure should be recorded, not returned: %v", err)
@@ -751,7 +772,7 @@ func TestDatasetService_Parse(t *testing.T) {
}
cols := []domain.AttributeColumn{{Name: "като"}, {Name: "F_2023"}}
parser := ColumnParser(func(string, []byte) ([]domain.AttributeColumn, error) { return cols, nil })
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, parser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, parser, noopRowParser, &stubConverter{})
if err := svc.Parse(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -775,7 +796,7 @@ func TestDatasetService_Parse_RecordsFailure(t *testing.T) {
parser := ColumnParser(func(string, []byte) ([]domain.AttributeColumn, error) {
return nil, errors.New("corrupt archive")
})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, parser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, parser, noopRowParser, &stubConverter{})
if err := svc.Parse(context.Background(), id); err != nil {
t.Fatalf("parse failure should be recorded, not returned: %v", err)
@@ -886,7 +907,7 @@ func TestDatasetService_SaveMapping(t *testing.T) {
repo := newStubDatasetRepo()
repo.store[id] = base
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
got, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: validYears})
if err != nil {
@@ -992,7 +1013,7 @@ func TestDatasetService_Extract(t *testing.T) {
geoms := []json.RawMessage{json.RawMessage(`{"type":"Polygon","coordinates":[[[76,43],[77,43],[77,44],[76,43]]]}`)}
geom := []byte(`{"type":"GeometryCollection","geometries":[` + string(geoms[0]) + `]}`)
conv := &stubConverter{vectorFeatures: geoms}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, conv)
if err := svc.Extract(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -1427,7 +1448,7 @@ func TestDatasetService_Reprocess_ClearsStaleGeometryAndGeoJSON(t *testing.T) {
GeoJSON: json.RawMessage(`{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"rows":[]}}]}`),
}
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
if _, err := svc.Reprocess(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -1454,7 +1475,7 @@ func TestDatasetService_ReprocessAll_FiltersByFileType(t *testing.T) {
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[1,1],[2,2],[3,1],[1,1]]]}`),
}
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
vectorType := domain.FileTypeVector
enqueued, failures, err := svc.ReprocessAll(context.Background(), &vectorType)
@@ -1738,9 +1759,11 @@ func TestDatasetService_Patch_NotFound(t *testing.T) {
func TestDatasetService_Upload_StoresPeriodAndGroup(t *testing.T) {
repo := newStubDatasetRepo()
svc := newDatasetService(repo, &stubStore{}, true)
groups := newStubGroupRepo()
svc := newDatasetServiceWithGroups(repo, newStubStore(), true, groups)
gid := uuid.New()
groups.store[gid] = domain.DatasetGroup{ID: gid, Code: "pop", Name: "Population"}
start, end := "2020-01-01", "2020-12-31"
in := validUpload()
in.DatasetGroupID = &gid
@@ -1772,3 +1795,134 @@ func TestDatasetService_Upload_RejectsReversedPeriod(t *testing.T) {
t.Fatalf("nothing should be uploaded on validation failure, got %v", store.put)
}
}
func TestDatasetService_DownloadSLD_FallsBackToGroup(t *testing.T) {
ctx := context.Background()
repo := newStubDatasetRepo()
store := newStubStore()
groups := newStubGroupRepo()
svc := newDatasetServiceWithGroups(repo, store, true, groups)
groupSLD := "groups/style.sld"
filename, contentType := "group.sld", "application/xml"
group := domain.DatasetGroup{
ID: uuid.New(), Code: "pop", Name: "Population",
SldStorageKey: &groupSLD, SldFilename: &filename, SldContentType: &contentType,
}
groups.store[group.ID] = group
store.objects[groupSLD] = []byte("<StyledLayerDescriptor/>")
id := uuid.New()
repo.store[id] = domain.Dataset{ID: id, DatasetGroupID: &group.ID}
dataset, obj, err := svc.DownloadSLD(ctx, id)
if err != nil {
t.Fatal(err)
}
defer obj.Close()
if dataset.SldStorageKey == nil || *dataset.SldStorageKey != groupSLD {
t.Fatalf("want group sld key %s, got %v", groupSLD, dataset.SldStorageKey)
}
if dataset.SldFilename == nil || *dataset.SldFilename != filename {
t.Fatalf("want group sld filename %s, got %v", filename, dataset.SldFilename)
}
body, err := io.ReadAll(obj)
if err != nil {
t.Fatal(err)
}
if string(body) != "<StyledLayerDescriptor/>" {
t.Fatalf("want the group's sld content, got %q", body)
}
}
func TestDatasetService_DownloadSLD_NoFallbackWhenGroupHasNoSLD(t *testing.T) {
repo := newStubDatasetRepo()
groups := newStubGroupRepo()
svc := newDatasetServiceWithGroups(repo, newStubStore(), true, groups)
group := domain.DatasetGroup{ID: uuid.New(), Code: "pop", Name: "Population"}
groups.store[group.ID] = group
id := uuid.New()
repo.store[id] = domain.Dataset{ID: id, DatasetGroupID: &group.ID}
if _, _, err := svc.DownloadSLD(context.Background(), id); !errors.Is(err, ErrSLDNotFound) {
t.Fatalf("want ErrSLDNotFound, got %v", err)
}
}
func TestDatasetService_Upload_RejectsUnknownGroup(t *testing.T) {
svc := newDatasetService(newStubDatasetRepo(), newStubStore(), true)
in := validUpload()
missing := uuid.New()
in.DatasetGroupID = &missing
if _, err := svc.Upload(context.Background(), in); !errors.Is(err, domain.ErrValidation) {
t.Fatalf("want ErrValidation, got %v", err)
}
}
func TestDatasetService_CreateRasterFromFile(t *testing.T) {
ctx := context.Background()
repo := newStubDatasetRepo()
store := newStubStore()
groups := newStubGroupRepo()
jobs := &stubEnqueuer{}
svc := NewDatasetService(repo, store, stubCategoryReader{exists: true}, groups, jobs, noopParser, noopRowParser, &stubConverter{})
group := domain.DatasetGroup{ID: uuid.New(), Code: "lp-z", Name: "Landscape Z"}
groups.store[group.ID] = group
// A minimal little-endian TIFF header is enough for content sniffing.
path := filepath.Join(t.TempDir(), "lp-z-integrated.tif")
if err := os.WriteFile(path, []byte("II*\x00 raster payload"), 0o600); err != nil {
t.Fatal(err)
}
end := "2024-12-31"
dataset, err := svc.CreateRasterFromFile(ctx, CreateRasterInput{
CategoryID: uuid.New(),
DatasetGroupID: &group.ID,
Name: "Landscape Z integrated",
Filename: "lp-z-integrated.tif",
Path: path,
EndDate: &end,
Automated: true,
})
if err != nil {
t.Fatal(err)
}
if dataset.FileType != domain.FileTypeRaster {
t.Fatalf("want raster file type, got %q", dataset.FileType)
}
if dataset.DatasetGroupID == nil || *dataset.DatasetGroupID != group.ID {
t.Fatalf("want group %s, got %v", group.ID, dataset.DatasetGroupID)
}
if dataset.EndDate == nil || *dataset.EndDate != end {
t.Fatalf("want end date %s, got %v", end, dataset.EndDate)
}
if !dataset.Automated {
t.Fatal("want automated dataset")
}
if dataset.SizeBytes != int64(len("II*\x00 raster payload")) {
t.Fatalf("want the file size, got %d", dataset.SizeBytes)
}
if len(store.put) != 1 {
t.Fatalf("want the raster stored once, got %v", store.put)
}
if len(jobs.converted) != 1 || jobs.converted[0] != dataset.ID {
t.Fatalf("want a convert job for %s, got %v", dataset.ID, jobs.converted)
}
}
func TestDatasetService_CreateRasterFromFile_MissingFile(t *testing.T) {
svc := newDatasetService(newStubDatasetRepo(), newStubStore(), true)
_, err := svc.CreateRasterFromFile(context.Background(), CreateRasterInput{
CategoryID: uuid.New(),
Filename: "missing.tif",
Path: filepath.Join(t.TempDir(), "missing.tif"),
})
if err == nil {
t.Fatal("want an error for a missing file")
}
}