infureal/gis
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Fix processing for vector attributres table

Browse Source
This commit is contained in:
Bakhtiyar Issakhmetov
2026-07-10 23:34:19 +05:00
parent 017f869380
commit 47b2e8653d
6 changed files with 315 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
internal/service/dataset_test.go
View File

@@ -7,6 +7,7 @@ import (
"errors"
"io"
"os"
"sort"
"strings"
"testing"
"time"
@@ -23,6 +24,7 @@ type stubDatasetRepo struct {
districts map[string]domain.District
createErr error
deleted []uuid.UUID
cleared []uuid.UUID
lastLimit, lastOffset int
lastFilter domain.DatasetFilter
}
@@ -58,7 +60,24 @@ func (r *stubDatasetRepo) ListSummaries(_ context.Context, filter domain.Dataset
r.lastFilter = filter
r.lastLimit = limit
r.lastOffset = offset
return nil, nil
var matched []domain.DatasetSummary
for _, d := range r.store {
if filter.FileType != nil && d.FileType != *filter.FileType {
continue
}
matched = append(matched, domain.DatasetSummary{ID: d.ID, FileType: d.FileType})
}
sort.Slice(matched, func(i, j int) bool { return matched[i].ID.String() < matched[j].ID.String() })
if offset >= len(matched) {
return nil, nil
}
end := offset + limit
if end > len(matched) {
end = len(matched)
}
return matched[offset:end], nil
}
func (r *stubDatasetRepo) Count(_ context.Context, _ domain.DatasetFilter) (int, error) {
@@ -146,6 +165,18 @@ func (r *stubDatasetRepo) SetProperties(_ context.Context, id uuid.UUID, propert
return nil
}
func (r *stubDatasetRepo) ClearGeometryAndGeoJSON(_ context.Context, id uuid.UUID) error {
d, ok := r.store[id]
if !ok {
return domain.ErrNotFound
}
d.Geometry = nil
d.GeoJSON = nil
r.store[id] = d
r.cleared = append(r.cleared, id)
return nil
}
func (r *stubDatasetRepo) SetGeoJSON(_ context.Context, id uuid.UUID, geojson []byte) error {
d, ok := r.store[id]
if !ok {
@@ -226,12 +257,12 @@ func (s *stubEnqueuer) EnqueueConvert(_ context.Context, id uuid.UUID) error {
// stubConverter records raster conversions.
type stubConverter struct {
cogCalls int
toCOGErr error
footprint []byte
footprintFn func(src string) ([]byte, error)
vectorGeom []byte
vectorGeomFn func(src string) ([]byte, error)
cogCalls int
toCOGErr error
footprint []byte
footprintFn func(src string) ([]byte, error)
vectorFeatures []json.RawMessage
vectorFeaturesFn func(src string) ([]json.RawMessage, error)
}
func (c *stubConverter) ToCOG(_ context.Context, _, dst string) error {
@@ -249,11 +280,11 @@ func (c *stubConverter) Footprint(_ context.Context, src string) ([]byte, error)
return c.footprint, nil
}
func (c *stubConverter) VectorGeometry(_ context.Context, src string) ([]byte, error) {
if c.vectorGeomFn != nil {
return c.vectorGeomFn(src)
func (c *stubConverter) VectorFeatures(_ context.Context, src string) ([]json.RawMessage, error) {
if c.vectorFeaturesFn != nil {
return c.vectorFeaturesFn(src)
}
return c.vectorGeom, nil
return c.vectorFeatures, nil
}
var (
@@ -471,8 +502,11 @@ func TestDatasetService_ExtractProperties(t *testing.T) {
{"name": "Almaty", "pop": "2000"},
}
rp := RowParser(func(string, []byte) ([]map[string]string, error) { return rows, nil })
geom := []byte(`{"type":"GeometryCollection","geometries":[]}`)
conv := &stubConverter{vectorGeom: geom}
geoms := []json.RawMessage{
json.RawMessage(`{"type":"Polygon","coordinates":[[[1,1],[2,2],[3,1],[1,1]]]}`),
json.RawMessage(`{"type":"Polygon","coordinates":[[[4,4],[5,5],[6,4],[4,4]]]}`),
}
conv := &stubConverter{vectorFeatures: geoms}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
if err := svc.ExtractProperties(context.Background(), id); err != nil {
@@ -489,8 +523,25 @@ func TestDatasetService_ExtractProperties(t *testing.T) {
if len(parsed) != 2 {
t.Fatalf("want 2 rows in properties, got %d", len(parsed))
}
if string(got.Geometry) != string(geom) {
t.Fatalf("want geometry %s, got %s", geom, got.Geometry)
wantGeometry := `{"type":"GeometryCollection","geometries":[` + string(geoms[0]) + `,` + string(geoms[1]) + `]}`
if string(got.Geometry) != wantGeometry {
t.Fatalf("want dissolved geometry %s, got %s", wantGeometry, got.Geometry)
}
// Regression: each polygon must keep its own attribute row as its Feature's
// properties, not a single dissolved geometry sharing every row.
var fc domain.FeatureCollection
if err := json.Unmarshal(got.GeoJSON, &fc); err != nil {
t.Fatalf("geojson not persisted / invalid: %v (%s)", err, got.GeoJSON)
}
if len(fc.Features) != 2 {
t.Fatalf("want 2 features (one per polygon), got %d: %s", len(fc.Features), got.GeoJSON)
}
if string(fc.Features[0].Geometry) != string(geoms[0]) || fc.Features[0].Properties["name"] != "Astana" {
t.Fatalf("feature 0 not paired with its own geometry/row: %+v", fc.Features[0])
}
if string(fc.Features[1].Geometry) != string(geoms[1]) || fc.Features[1].Properties["name"] != "Almaty" {
t.Fatalf("feature 1 not paired with its own geometry/row: %+v", fc.Features[1])
}
}
@@ -505,7 +556,7 @@ func TestDatasetService_ExtractProperties_GeometryBestEffort(t *testing.T) {
return []map[string]string{{"name": "Astana"}}, nil
})
// Geometry extraction fails; the job must still succeed with properties set.
conv := &stubConverter{vectorGeomFn: func(string) ([]byte, error) {
conv := &stubConverter{vectorFeaturesFn: func(string) ([]json.RawMessage, error) {
return nil, errors.New("ogr2ogr boom")
}}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
@@ -800,8 +851,9 @@ func TestDatasetService_Extract(t *testing.T) {
}
rows := []map[string]string{{"като": "751010000", "F_2023": "100"}}
rp := RowParser(func(string, []byte) ([]map[string]string, error) { return rows, nil })
geom := []byte(`{"type":"GeometryCollection","geometries":[]}`)
conv := &stubConverter{vectorGeom: geom}
geoms := []json.RawMessage{json.RawMessage(`{"type":"Polygon","coordinates":[[[76,43],[77,43],[77,44],[76,43]]]}`)}
geom := []byte(`{"type":"GeometryCollection","geometries":[` + string(geoms[0]) + `]}`)
conv := &stubConverter{vectorFeatures: geoms}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
if err := svc.Extract(context.Background(), id); err != nil {
@@ -1166,6 +1218,69 @@ func TestDatasetService_WaitForStatus(t *testing.T) {
})
}
func TestDatasetService_Reprocess_ClearsStaleGeometryAndGeoJSON(t *testing.T) {
repo := newStubDatasetRepo()
id := uuid.New()
repo.store[id] = domain.Dataset{
ID: id, FileType: domain.FileTypeVector, Status: domain.DatasetStatusReady,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[1,1],[2,2],[3,1],[1,1]]]}`),
GeoJSON: json.RawMessage(`{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"rows":[]}}]}`),
}
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
if _, err := svc.Reprocess(context.Background(), id); err != nil {
t.Fatalf("unexpected error: %v", err)
}
got := repo.store[id]
if got.Geometry != nil || got.GeoJSON != nil {
t.Fatalf("want stale geometry/geojson cleared, got geometry=%s geojson=%s", got.Geometry, got.GeoJSON)
}
if len(enq.properties) != 1 || enq.properties[0] != id {
t.Fatalf("want properties job enqueued for %s, got %+v", id, enq.properties)
}
}
func TestDatasetService_ReprocessAll_FiltersByFileType(t *testing.T) {
repo := newStubDatasetRepo()
vectorID, rasterID := uuid.New(), uuid.New()
repo.store[vectorID] = domain.Dataset{
ID: vectorID, FileType: domain.FileTypeVector,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[1,1],[2,2],[3,1],[1,1]]]}`),
GeoJSON: json.RawMessage(`{"type":"FeatureCollection","features":[]}`),
}
repo.store[rasterID] = domain.Dataset{
ID: rasterID, FileType: domain.FileTypeRaster,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[1,1],[2,2],[3,1],[1,1]]]}`),
}
enq := &stubEnqueuer{}
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, enq, noopParser, noopRowParser, &stubConverter{})
vectorType := domain.FileTypeVector
enqueued, failures, err := svc.ReprocessAll(context.Background(), &vectorType)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(failures) != 0 {
t.Fatalf("unexpected failures: %+v", failures)
}
if enqueued != 1 {
t.Fatalf("want 1 dataset enqueued, got %d", enqueued)
}
if len(enq.properties) != 1 || enq.properties[0] != vectorID {
t.Fatalf("want only the vector dataset's properties job enqueued, got %+v", enq.properties)
}
if len(enq.converted) != 0 {
t.Fatalf("raster dataset must not be reprocessed when --type=vector, got %+v", enq.converted)
}
if got := repo.store[vectorID]; got.Geometry != nil || got.GeoJSON != nil {
t.Fatalf("vector dataset's stale geometry/geojson must be cleared, got geometry=%s geojson=%s", got.Geometry, got.GeoJSON)
}
if got := repo.store[rasterID]; got.Geometry == nil {
t.Fatalf("raster dataset outside the --type filter must be left untouched")
}
}
func TestDatasetService_Delete_RemovesObject(t *testing.T) {
repo := newStubDatasetRepo()
id := uuid.New()