fix: Fix geometry parsing
This commit is contained in:
@@ -33,7 +33,7 @@ type DatasetRepository interface {
|
||||
MarkParseFailed(ctx context.Context, id uuid.UUID, reason string) error
|
||||
MarkReady(ctx context.Context, id uuid.UUID) error
|
||||
MarkConverted(ctx context.Context, id uuid.UUID, cogKey string, footprint []byte) error
|
||||
SetProperties(ctx context.Context, id uuid.UUID, properties []byte) error
|
||||
SetProperties(ctx context.Context, id uuid.UUID, properties, geometry []byte) error
|
||||
SaveMapping(ctx context.Context, id uuid.UUID, katoColumn string, years []domain.YearColumn) (domain.Dataset, error)
|
||||
ReplaceObservations(ctx context.Context, datasetID uuid.UUID, obs []domain.Observation) error
|
||||
ListObservations(ctx context.Context, datasetID uuid.UUID, katoCode *string, limit, offset int) ([]domain.Observation, error)
|
||||
@@ -82,11 +82,15 @@ type ColumnParser func(filename string, data []byte) ([]domain.AttributeColumn,
|
||||
// RowParser reads every attribute row from a file's raw bytes as name->value maps.
|
||||
type RowParser func(filename string, data []byte) ([]map[string]string, error)
|
||||
|
||||
// RasterConverter converts a raster file to a Cloud-Optimized GeoTIFF and reads
|
||||
// its footprint. It operates on local file paths.
|
||||
// RasterConverter converts a raster file to a Cloud-Optimized GeoTIFF, reads its
|
||||
// footprint, and dissolves a vector file's features into a single geometry. It
|
||||
// operates on local file paths.
|
||||
type RasterConverter interface {
|
||||
ToCOG(ctx context.Context, srcPath, dstPath string) error
|
||||
Footprint(ctx context.Context, srcPath string) ([]byte, error)
|
||||
// VectorGeometry returns the combined geometry of a vector file's features as
|
||||
// GeoJSON in EPSG:4326 (nil if the file has no features).
|
||||
VectorGeometry(ctx context.Context, srcPath string) ([]byte, error)
|
||||
}
|
||||
|
||||
// UploadInput carries everything needed to store a new dataset.
|
||||
@@ -249,10 +253,12 @@ func (s *DatasetService) Reprocess(ctx context.Context, id uuid.UUID) (domain.Da
|
||||
return dataset, nil
|
||||
}
|
||||
|
||||
// ExtractProperties reads a plain vector dataset's attribute table and stores it
|
||||
// (as a JSON array of row objects) in the properties column, then marks the
|
||||
// dataset ready. Invoked by the worker. Parse failures are recorded; storage
|
||||
// failures are returned for retry.
|
||||
// ExtractProperties reads a plain vector dataset's attribute table and spatial
|
||||
// geometry and stores them (the attribute table as a JSON array of row objects
|
||||
// in the properties column, the dissolved feature geometry in the geometry
|
||||
// column), then marks the dataset ready. Invoked by the worker. Parse failures
|
||||
// are recorded; storage failures are returned for retry. Geometry extraction is
|
||||
// best-effort: a failure leaves geometry unset rather than failing the job.
|
||||
func (s *DatasetService) ExtractProperties(ctx context.Context, id uuid.UUID) error {
|
||||
dataset, err := s.repo.GetByID(ctx, id)
|
||||
if err != nil {
|
||||
@@ -278,7 +284,31 @@ func (s *DatasetService) ExtractProperties(ctx context.Context, id uuid.UUID) er
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.repo.SetProperties(ctx, id, properties)
|
||||
|
||||
geometry := s.vectorGeometry(ctx, dataset.Filename, data)
|
||||
return s.repo.SetProperties(ctx, id, properties, geometry)
|
||||
}
|
||||
|
||||
// vectorGeometry dissolves a vector file's features into a single GeoJSON
|
||||
// geometry via the converter. It writes the in-memory bytes to a temp file
|
||||
// (preserving the extension so the converter detects the format) because the
|
||||
// converter operates on file paths. Best-effort: any failure yields nil.
|
||||
func (s *DatasetService) vectorGeometry(ctx context.Context, filename string, data []byte) []byte {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
f, err := os.CreateTemp("", "gis-vec-*"+ext)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
if _, err := f.Write(data); err != nil {
|
||||
f.Close()
|
||||
return nil
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return nil
|
||||
}
|
||||
geom, _ := s.converter.VectorGeometry(ctx, f.Name())
|
||||
return geom
|
||||
}
|
||||
|
||||
// hasAttributeData reports whether any row carries at least one attribute.
|
||||
|
||||
@@ -125,12 +125,15 @@ func (r *stubDatasetRepo) MarkConverted(_ context.Context, id uuid.UUID, cogKey
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *stubDatasetRepo) SetProperties(_ context.Context, id uuid.UUID, properties []byte) error {
|
||||
func (r *stubDatasetRepo) SetProperties(_ context.Context, id uuid.UUID, properties, geometry []byte) error {
|
||||
d, ok := r.store[id]
|
||||
if !ok {
|
||||
return domain.ErrNotFound
|
||||
}
|
||||
d.Properties = properties
|
||||
if len(geometry) > 0 {
|
||||
d.Geometry = geometry
|
||||
}
|
||||
d.Status = domain.DatasetStatusReady
|
||||
r.store[id] = d
|
||||
return nil
|
||||
@@ -192,10 +195,12 @@ func (s *stubEnqueuer) EnqueueConvert(_ context.Context, id uuid.UUID) error {
|
||||
|
||||
// stubConverter records raster conversions.
|
||||
type stubConverter struct {
|
||||
cogCalls int
|
||||
toCOGErr error
|
||||
footprint []byte
|
||||
footprintFn func(src string) ([]byte, error)
|
||||
cogCalls int
|
||||
toCOGErr error
|
||||
footprint []byte
|
||||
footprintFn func(src string) ([]byte, error)
|
||||
vectorGeom []byte
|
||||
vectorGeomFn func(src string) ([]byte, error)
|
||||
}
|
||||
|
||||
func (c *stubConverter) ToCOG(_ context.Context, _, dst string) error {
|
||||
@@ -213,6 +218,13 @@ func (c *stubConverter) Footprint(_ context.Context, src string) ([]byte, error)
|
||||
return c.footprint, nil
|
||||
}
|
||||
|
||||
func (c *stubConverter) VectorGeometry(_ context.Context, src string) ([]byte, error) {
|
||||
if c.vectorGeomFn != nil {
|
||||
return c.vectorGeomFn(src)
|
||||
}
|
||||
return c.vectorGeom, nil
|
||||
}
|
||||
|
||||
var (
|
||||
noopParser ColumnParser = func(string, []byte) ([]domain.AttributeColumn, error) { return nil, nil }
|
||||
noopRowParser RowParser = func(string, []byte) ([]map[string]string, error) { return nil, nil }
|
||||
@@ -428,7 +440,9 @@ func TestDatasetService_ExtractProperties(t *testing.T) {
|
||||
{"name": "Almaty", "pop": "2000"},
|
||||
}
|
||||
rp := RowParser(func(string, []byte) ([]map[string]string, error) { return rows, nil })
|
||||
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, &stubConverter{})
|
||||
geom := []byte(`{"type":"GeometryCollection","geometries":[]}`)
|
||||
conv := &stubConverter{vectorGeom: geom}
|
||||
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
|
||||
|
||||
if err := svc.ExtractProperties(context.Background(), id); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -444,6 +458,40 @@ func TestDatasetService_ExtractProperties(t *testing.T) {
|
||||
if len(parsed) != 2 {
|
||||
t.Fatalf("want 2 rows in properties, got %d", len(parsed))
|
||||
}
|
||||
if string(got.Geometry) != string(geom) {
|
||||
t.Fatalf("want geometry %s, got %s", geom, got.Geometry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_ExtractProperties_GeometryBestEffort(t *testing.T) {
|
||||
id := uuid.New()
|
||||
repo := newStubDatasetRepo()
|
||||
repo.store[id] = domain.Dataset{
|
||||
ID: id, FileType: domain.FileTypeVector,
|
||||
Filename: "d.geojson", StorageKey: "k", Status: domain.DatasetStatusProcessing,
|
||||
}
|
||||
rp := RowParser(func(string, []byte) ([]map[string]string, error) {
|
||||
return []map[string]string{{"name": "Astana"}}, nil
|
||||
})
|
||||
// Geometry extraction fails; the job must still succeed with properties set.
|
||||
conv := &stubConverter{vectorGeomFn: func(string) ([]byte, error) {
|
||||
return nil, errors.New("ogr2ogr boom")
|
||||
}}
|
||||
svc := NewDatasetService(repo, &stubStore{}, stubCategoryReader{exists: true}, &stubEnqueuer{}, noopParser, rp, conv)
|
||||
|
||||
if err := svc.ExtractProperties(context.Background(), id); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
got := repo.store[id]
|
||||
if got.Status != domain.DatasetStatusReady {
|
||||
t.Fatalf("want ready, got %q", got.Status)
|
||||
}
|
||||
if got.Geometry != nil {
|
||||
t.Fatalf("expected nil geometry on extraction failure, got %s", got.Geometry)
|
||||
}
|
||||
if got.Properties == nil {
|
||||
t.Fatalf("expected properties to be set despite geometry failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetService_ExtractProperties_NoTable(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user