infureal/gis
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dataset-groups): CRUD, latest-in-group, group SLD with dataset fallback, CreateRasterFromFile

Browse Source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Bakhtiyar Issakhmetov
2026-09-05 13:29:01 +05:00
parent 32eba016b3
commit bc524f3d98
11 changed files with 1366 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

192
internal/service/dataset_test.go
View File

@@ -7,6 +7,7 @@ import (
"errors"
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"
@@ -358,22 +359,36 @@ var (
noopRowParser RowParser = func(string, []byte) ([]map[string]string, error) { return nil, nil }
)
// stubStore records object-store interactions.
// stubStore records object-store interactions and keeps stored bodies so tests
// can read them back.
type stubStore struct {
put []string
removed []string
putErr error
objects map[string][]byte
}
func (s *stubStore) Put(_ context.Context, key string, _ io.Reader, _ int64, _ string) error {
func newStubStore() *stubStore { return &stubStore{objects: map[string][]byte{}} }
func (s *stubStore) Put(_ context.Context, key string, r io.Reader, _ int64, _ string) error {
if s.putErr != nil {
return s.putErr
}
s.put = append(s.put, key)
if s.objects != nil {
body, err := io.ReadAll(r)
if err != nil {
return err
}
s.objects[key] = body
}
return nil
}
func (s *stubStore) Get(_ context.Context, _ string) (io.ReadCloser, error) {
func (s *stubStore) Get(_ context.Context, key string) (io.ReadCloser, error) {
if body, ok := s.objects[key]; ok {
return io.NopCloser(bytes.NewReader(body)), nil
}
return io.NopCloser(strings.NewReader("")), nil
}
@@ -414,7 +429,13 @@ func validUpload() UploadInput {
}
func newDatasetService(repo *stubDatasetRepo, store *stubStore, catExists bool) *DatasetService {
return NewDatasetService(repo, store, stubCategoryReader{exists: catExists}, &stubEnqueuer{}, noopParser, noopRowParser, &stubConverter{})
return newDatasetServiceWithGroups(repo, store, catExists, newStubGroupRepo())
}
// newDatasetServiceWithGroups builds a DatasetService over a caller-supplied
// group repository, for tests that need a pre-populated dataset group.
func newDatasetServiceWithGroups(repo *stubDatasetRepo, store *stubStore, catExists bool, groups *stubGroupRepo) *DatasetService {
return NewDatasetService(repo, store, stubCategoryReader{exists: catExists}, groups, &stubEnqueuer{}, noopParser, noopRowParser, &stubConverter{})
}
func TestDatasetService_Upload_Validation(t *testing.T) {
@@ -521,7 +542,7 @@ func TestDatasetService_Upload_CompensatesOnDBFailure(t *testing.T) {
func TestDatasetService_Upload_VectorWithKato_EnqueuesParse(t *testing.T) {
repo := newStubDatasetRepo()
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
in := validUpload()
in.FileType = domain.FileTypeVectorWithKato
@@ -542,7 +563,7 @@ func TestDatasetService_Upload_VectorWithKato_EnqueuesParse(t *testing.T) {
func TestDatasetService_Upload_Vector_EnqueuesProperties(t *testing.T) {
repo := newStubDatasetRepo()
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
got, err := svc.Upload(context.Background(), validUpload()) // plain vector .geojson
if err != nil {
@@ -573,7 +594,7 @@ func TestDatasetService_ExtractProperties(t *testing.T) {
json.RawMessage(`{"type":"Polygon","coordinates":[[[4,4],[5,5],[6,4],[4,4]]]}`),
}
conv := &stubConverter{vectorFeatures: geoms}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, conv)
if err := svc.ExtractProperties(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -625,7 +646,7 @@ func TestDatasetService_ExtractProperties_GeometryBestEffort(t *testing.T) {
conv := &stubConverter{vectorFeaturesFn: func(string) ([]json.RawMessage, error) {
return nil, errors.New("ogr2ogr boom")
}}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, conv)
if err := svc.ExtractProperties(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -653,7 +674,7 @@ func TestDatasetService_ExtractProperties_NoTable(t *testing.T) {
rp := RowParser(func(string, []byte) ([]map[string]string, error) {
return []map[string]string{{}, {}}, nil
})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, &stubConverter{})
if err := svc.ExtractProperties(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -670,7 +691,7 @@ func TestDatasetService_ExtractProperties_NoTable(t *testing.T) {
func TestDatasetService_Upload_Raster_EnqueuesConvert(t *testing.T) {
repo := newStubDatasetRepo()
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
in := validUpload()
in.FileType = domain.FileTypeRaster
@@ -700,7 +721,7 @@ func TestDatasetService_ConvertToCOG(t *testing.T) {
store := &stubStore{}
footprint := []byte(`{"type":"Polygon","coordinates":[[[70,50],[72,50],[72,52],[70,52],[70,50]]]}`)
conv := &stubConverter{footprint: footprint}
svc := NewDatasetService(repo, store, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, noopRowParser, conv)
svc := NewDatasetService(repo, store, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, noopRowParser, conv)
if err := svc.ConvertToCOG(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -732,7 +753,7 @@ func TestDatasetService_ConvertToCOG_RecordsFailure(t *testing.T) {
Status: domain.DatasetStatusProcessing,
}
conv := &stubConverter{toCOGErr: errors.New("gdal failed")}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, noopRowParser, conv)
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, noopRowParser, conv)
if err := svc.ConvertToCOG(context.Background(), id); err != nil {
t.Fatalf("conversion failure should be recorded, not returned: %v", err)
@@ -751,7 +772,7 @@ func TestDatasetService_Parse(t *testing.T) {
}
cols := []domain.AttributeColumn{{Name: "като"}, {Name: "F_2023"}}
parser := ColumnParser(func(string, []byte) ([]domain.AttributeColumn, error) { return cols, nil })
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, parser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, parser, noopRowParser, &stubConverter{})
if err := svc.Parse(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -775,7 +796,7 @@ func TestDatasetService_Parse_RecordsFailure(t *testing.T) {
parser := ColumnParser(func(string, []byte) ([]domain.AttributeColumn, error) {
return nil, errors.New("corrupt archive")
})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, parser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, parser, noopRowParser, &stubConverter{})
if err := svc.Parse(context.Background(), id); err != nil {
t.Fatalf("parse failure should be recorded, not returned: %v", err)
@@ -886,7 +907,7 @@ func TestDatasetService_SaveMapping(t *testing.T) {
repo := newStubDatasetRepo()
repo.store[id] = base
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
got, err := svc.SaveMapping(ctx, id, MappingInput{KatoColumn: "като", YearColumns: validYears})
if err != nil {
@@ -992,7 +1013,7 @@ func TestDatasetService_Extract(t *testing.T) {
geoms := []json.RawMessage{json.RawMessage(`{"type":"Polygon","coordinates":[[[76,43],[77,43],[77,44],[76,43]]]}`)}
geom := []byte(`{"type":"GeometryCollection","geometries":[` + string(geoms[0]) + `]}`)
conv := &stubConverter{vectorFeatures: geoms}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), &stubEnqueuer{}, noopParser, rp, conv)
if err := svc.Extract(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -1427,7 +1448,7 @@ func TestDatasetService_Reprocess_ClearsStaleGeometryAndGeoJSON(t *testing.T) {
GeoJSON: json.RawMessage(`{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"rows":[]}}]}`),
}
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
if _, err := svc.Reprocess(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -1454,7 +1475,7 @@ func TestDatasetService_ReprocessAll_FiltersByFileType(t *testing.T) {
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[1,1],[2,2],[3,1],[1,1]]]}`),
}
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, newStubGroupRepo(), enq, noopParser, noopRowParser, &stubConverter{})
vectorType := domain.FileTypeVector
enqueued, failures, err := svc.ReprocessAll(context.Background(), &vectorType)
@@ -1738,9 +1759,11 @@ func TestDatasetService_Patch_NotFound(t *testing.T) {
func TestDatasetService_Upload_StoresPeriodAndGroup(t *testing.T) {
repo := newStubDatasetRepo()
svc := newDatasetService(repo, &stubStore{}, true)
groups := newStubGroupRepo()
svc := newDatasetServiceWithGroups(repo, newStubStore(), true, groups)
gid := uuid.New()
groups.store[gid] = domain.DatasetGroup{ID: gid, Code: "pop", Name: "Population"}
start, end := "2020-01-01", "2020-12-31"
in := validUpload()
in.DatasetGroupID = &gid
@@ -1772,3 +1795,134 @@ func TestDatasetService_Upload_RejectsReversedPeriod(t *testing.T) {
t.Fatalf("nothing should be uploaded on validation failure, got %v", store.put)
}
}
func TestDatasetService_DownloadSLD_FallsBackToGroup(t *testing.T) {
ctx := context.Background()
repo := newStubDatasetRepo()
store := newStubStore()
groups := newStubGroupRepo()
svc := newDatasetServiceWithGroups(repo, store, true, groups)
groupSLD := "groups/style.sld"
filename, contentType := "group.sld", "application/xml"
group := domain.DatasetGroup{
ID: uuid.New(), Code: "pop", Name: "Population",
SldStorageKey: &groupSLD, SldFilename: &filename, SldContentType: &contentType,
}
groups.store[group.ID] = group
store.objects[groupSLD] = []byte("<StyledLayerDescriptor/>")
id := uuid.New()
repo.store[id] = domain.Dataset{ID: id, DatasetGroupID: &group.ID}
dataset, obj, err := svc.DownloadSLD(ctx, id)
if err != nil {
t.Fatal(err)
}
defer obj.Close()
if dataset.SldStorageKey == nil || *dataset.SldStorageKey != groupSLD {
t.Fatalf("want group sld key %s, got %v", groupSLD, dataset.SldStorageKey)
}
if dataset.SldFilename == nil || *dataset.SldFilename != filename {
t.Fatalf("want group sld filename %s, got %v", filename, dataset.SldFilename)
}
body, err := io.ReadAll(obj)
if err != nil {
t.Fatal(err)
}
if string(body) != "<StyledLayerDescriptor/>" {
t.Fatalf("want the group's sld content, got %q", body)
}
}
func TestDatasetService_DownloadSLD_NoFallbackWhenGroupHasNoSLD(t *testing.T) {
repo := newStubDatasetRepo()
groups := newStubGroupRepo()
svc := newDatasetServiceWithGroups(repo, newStubStore(), true, groups)
group := domain.DatasetGroup{ID: uuid.New(), Code: "pop", Name: "Population"}
groups.store[group.ID] = group
id := uuid.New()
repo.store[id] = domain.Dataset{ID: id, DatasetGroupID: &group.ID}
if _, _, err := svc.DownloadSLD(context.Background(), id); !errors.Is(err, ErrSLDNotFound) {
t.Fatalf("want ErrSLDNotFound, got %v", err)
}
}
func TestDatasetService_Upload_RejectsUnknownGroup(t *testing.T) {
svc := newDatasetService(newStubDatasetRepo(), newStubStore(), true)
in := validUpload()
missing := uuid.New()
in.DatasetGroupID = &missing
if _, err := svc.Upload(context.Background(), in); !errors.Is(err, domain.ErrValidation) {
t.Fatalf("want ErrValidation, got %v", err)
}
}
func TestDatasetService_CreateRasterFromFile(t *testing.T) {
ctx := context.Background()
repo := newStubDatasetRepo()
store := newStubStore()
groups := newStubGroupRepo()
jobs := &stubEnqueuer{}
svc := NewDatasetService(repo, store, stubCategoryReader{exists: true}, groups, jobs, noopParser, noopRowParser, &stubConverter{})
group := domain.DatasetGroup{ID: uuid.New(), Code: "lp-z", Name: "Landscape Z"}
groups.store[group.ID] = group
// A minimal little-endian TIFF header is enough for content sniffing.
path := filepath.Join(t.TempDir(), "lp-z-integrated.tif")
if err := os.WriteFile(path, []byte("II*\x00 raster payload"), 0o600); err != nil {
t.Fatal(err)
}
end := "2024-12-31"
dataset, err := svc.CreateRasterFromFile(ctx, CreateRasterInput{
CategoryID: uuid.New(),
DatasetGroupID: &group.ID,
Name: "Landscape Z integrated",
Filename: "lp-z-integrated.tif",
Path: path,
EndDate: &end,
Automated: true,
})
if err != nil {
t.Fatal(err)
}
if dataset.FileType != domain.FileTypeRaster {
t.Fatalf("want raster file type, got %q", dataset.FileType)
}
if dataset.DatasetGroupID == nil || *dataset.DatasetGroupID != group.ID {
t.Fatalf("want group %s, got %v", group.ID, dataset.DatasetGroupID)
}
if dataset.EndDate == nil || *dataset.EndDate != end {
t.Fatalf("want end date %s, got %v", end, dataset.EndDate)
}
if !dataset.Automated {
t.Fatal("want automated dataset")
}
if dataset.SizeBytes != int64(len("II*\x00 raster payload")) {
t.Fatalf("want the file size, got %d", dataset.SizeBytes)
}
if len(store.put) != 1 {
t.Fatalf("want the raster stored once, got %v", store.put)
}
if len(jobs.converted) != 1 || jobs.converted[0] != dataset.ID {
t.Fatalf("want a convert job for %s, got %v", dataset.ID, jobs.converted)
}
}
func TestDatasetService_CreateRasterFromFile_MissingFile(t *testing.T) {
svc := newDatasetService(newStubDatasetRepo(), newStubStore(), true)
_, err := svc.CreateRasterFromFile(context.Background(), CreateRasterInput{
CategoryID: uuid.New(),
Filename: "missing.tif",
Path: filepath.Join(t.TempDir(), "missing.tif"),
})
if err == nil {
t.Fatal("want an error for a missing file")
}
}