fix(landscape): bound Compute's per-worker memory and clean up on failure

Review round 1: size per-worker buffers off the largest actual window
instead of WindowSize^2 (avoids allocating ~200+ MiB per worker for
small/partial rasters), cap workers at 8 regardless of GOMAXPROCS, remove
partially-written outputs when a run fails, wrap input-open errors with
node/group context, guard checkGrids against a node with no inputs, and
clamp a non-positive WindowSize so raster.Windows can't loop forever.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
2026-09-05 13:30:23 +05:00
parent bc524f3d98
commit ff6d5fc412
3 changed files with 209 additions and 12 deletions

View File

@@ -12,11 +12,19 @@ import (
type fakeIO struct {
mu sync.Mutex
files map[string]*fakeRaster
// failReadAfter, when set for a path, makes every ReadFloat64 call
// against that path fail once the cumulative read count (across all
// readers/workers) exceeds the configured threshold. Used to simulate a
// mid-run I/O failure.
failReadAfter map[string]int
readCounts map[string]int
}
type fakeRaster struct {
grid raster.Grid
data []float64 // row-major
opts raster.WriterOptions
}
func newFakeIO() *fakeIO { return &fakeIO{files: map[string]*fakeRaster{}} }
@@ -25,12 +33,39 @@ func (f *fakeIO) put(path string, g raster.Grid, data []float64) {
f.files[path] = &fakeRaster{grid: g, data: data}
}
type fakeReader struct{ r *fakeRaster }
// failReadAfterN makes ReadFloat64 against path fail starting from the
// (n+1)-th call to it.
func (f *fakeIO) failReadAfterN(path string, n int) {
f.mu.Lock()
defer f.mu.Unlock()
if f.failReadAfter == nil {
f.failReadAfter = map[string]int{}
f.readCounts = map[string]int{}
}
f.failReadAfter[path] = n
}
type fakeReader struct {
f *fakeIO
path string
r *fakeRaster
}
func (r *fakeReader) Grid() raster.Grid { return r.r.grid }
func (r *fakeReader) NoData() (float64, bool) { return -9999, true }
func (r *fakeReader) Close() error { return nil }
func (r *fakeReader) ReadFloat64(w raster.Window, dst []float64) error {
if r.f != nil {
r.f.mu.Lock()
if n, ok := r.f.failReadAfter[r.path]; ok {
r.f.readCounts[r.path]++
if r.f.readCounts[r.path] > n {
r.f.mu.Unlock()
return fmt.Errorf("fake: simulated read failure for %s", r.path)
}
}
r.f.mu.Unlock()
}
for y := 0; y < w.Height; y++ {
copy(dst[y*w.Width:(y+1)*w.Width], r.r.data[(w.Y+y)*r.r.grid.Width+w.X:(w.Y+y)*r.r.grid.Width+w.X+w.Width])
}
@@ -61,13 +96,13 @@ func (f *fakeIO) Open(_ context.Context, path string) (raster.Reader, error) {
if !ok {
return nil, fmt.Errorf("fake: %s not found", path)
}
return &fakeReader{r: r}, nil
return &fakeReader{f: f, path: path, r: r}, nil
}
func (f *fakeIO) Create(_ context.Context, path string, g raster.Grid, _ raster.WriterOptions) (raster.Writer, error) {
func (f *fakeIO) Create(_ context.Context, path string, g raster.Grid, o raster.WriterOptions) (raster.Writer, error) {
f.mu.Lock()
defer f.mu.Unlock()
r := &fakeRaster{grid: g, data: make([]float64, g.Width*g.Height)}
r := &fakeRaster{grid: g, data: make([]float64, g.Width*g.Height), opts: o}
f.files[path] = r
return &fakeWriter{f: f, r: r}, nil
}