fix(landscape): guard redelivery, detach terminal writes, bound workers

F2: Execute had no status guard, so a redelivered or stale message reran a
finished run. Runs that are neither queued nor running are logged and acked
without work; "running" still executes so a worker that crashed mid-run
recovers.

F4: failRun, finishNodeFailed and MarkRunFailed wrote the terminal status on
the caller's context, which is exactly the context that is already cancelled
when the run failed because of a shutdown or a sibling node — the write was
refused and the run stayed stuck in "running". They now write on a
context.WithoutCancel copy with a 10s timeout; the consumer's best-effort
MarkRunFailed does the same.

F5: peak memory scaled with the host's core count. LP_MAX_WORKERS (default 8)
is now the total window-worker budget of a run and LP_MAX_NODES (default 2) the
node fan-out; each node gets max(1, workers/nodes) through the new
landscape.ComputeWithWorkers. Compute and service.ComputeFunc keep their
signatures: Compute delegates with MaxWorkersPerNode, and a legacy ComputeFunc
handed to NewLandscapeService is wrapped.

F6: the stage loop used a plain errgroup, so a failing node let its siblings
run to completion. It now uses errgroup.WithContext and passes the group
context into runNode, cancelling siblings mid-compute.

F7: wrap every float32 product in an explicit conversion so the compiler
cannot fuse it into an FMA, keeping the kernels bit-identical to numpy's
per-op rounding.

F8: README's builder package list now matches the Dockerfile (adds g++), and
the Landscape section documents the memory model and the two new knobs.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
2026-09-05 14:44:34 +05:00
parent bcda76003c
commit 4051506b95
12 changed files with 563 additions and 69 deletions

View File

@@ -13,13 +13,27 @@ import (
"golang.org/x/sync/errgroup"
)
// Compute runs one node over its input rasters. See the package doc for the
// concurrency model: windows are processed by a worker pool; each worker owns
// MaxWorkersPerNode is the per-node worker cap Compute uses. Peak memory is
// roughly nodes x workers x window^2 x (inputs+outputs), so callers that run
// several nodes at once should divide their total worker budget between them
// and call ComputeWithWorkers instead.
const MaxWorkersPerNode = 8
// Compute runs one node over its input rasters with the default per-node
// worker cap (MaxWorkersPerNode). See ComputeWithWorkers.
func Compute(ctx context.Context, rio raster.IO, node Node, inputs map[string]string, outDir string, p Params) (map[string]string, error) {
return ComputeWithWorkers(ctx, rio, node, inputs, outDir, p, MaxWorkersPerNode)
}
// ComputeWithWorkers runs one node over its input rasters with an explicit
// worker cap. See the package doc for the concurrency model: windows are
// processed by a worker pool of at most workers goroutines; each worker owns
// its own Reader handles (GDAL datasets are not goroutine-safe) and writes are
// serialised per output. Window order does not affect the result. On any
// error, no output files are left behind: Compute best-effort removes every
// output it created before returning.
func Compute(ctx context.Context, rio raster.IO, node Node, inputs map[string]string, outDir string, p Params) (map[string]string, error) {
// error, no output files are left behind: ComputeWithWorkers best-effort
// removes every output it created before returning. A workers value below 1 is
// treated as 1.
func ComputeWithWorkers(ctx context.Context, rio raster.IO, node Node, inputs map[string]string, outDir string, p Params, workers int) (map[string]string, error) {
p = p.WithDefaults()
if p.WindowSize <= 0 {
// WithDefaults only fills in a zero value; guard against a negative
@@ -75,8 +89,8 @@ func Compute(ctx context.Context, rio raster.IO, node Node, inputs map[string]st
// per-window input/output buffers, and nodes can have many inputs and
// outputs (e.g. node 20: 6 inputs, 5 outputs), so an unbounded worker
// count multiplies memory with core count and with concurrently running
// nodes.
workers := min(runtime.GOMAXPROCS(0), len(windows), 8)
// nodes. The caller's budget is the third bound.
workers = min(runtime.GOMAXPROCS(0), len(windows), max(1, workers))
for w := 0; w < workers; w++ {
eg.Go(func() error {
readers, err := openAll(gctx, rio, node, paths)

View File

@@ -217,3 +217,36 @@ func TestRemoveOutputs_DeletesFiles(t *testing.T) {
}
}
}
// TestComputeWithWorkers_RespectsCap checks the caller's worker budget bounds
// the pool: each pool worker opens one reader per input and holds a buffer per
// input and output, so the open count bounds peak memory. checkGrids opens one
// extra set of handles before the pool starts, hence the (workers+1) bound.
func TestComputeWithWorkers_RespectsCap(t *testing.T) {
node, _ := NodeByCode("10")
inputs := len(node.Inputs)
for _, workers := range []int{1, 2, 4, 0, -3} {
t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) {
fio := newFakeIO()
g := grid(64, 64)
fio.put("z.tif", g, constRaster(64, 64, 2))
fio.put("u.tif", g, constRaster(64, 64, 4))
p := DefaultParams()
p.WindowSize = 8 // 64 windows, so the pool is never window-bound
if _, err := ComputeWithWorkers(context.Background(), fio, node,
map[string]string{OutZIntegrated: "z.tif", OutUIntegrated: "u.tif"}, "/out", p, workers); err != nil {
t.Fatal(err)
}
effective := max(1, workers)
if got, limit := fio.openCount(), (effective+1)*inputs; got > limit {
t.Fatalf("opened %d readers with workers=%d, want at most %d", got, workers, limit)
}
if fio.openCount() < inputs {
t.Fatalf("opened %d readers, want at least one set of %d", fio.openCount(), inputs)
}
})
}
}

View File

@@ -19,6 +19,17 @@ type fakeIO struct {
// mid-run I/O failure.
failReadAfter map[string]int
readCounts map[string]int
// opens counts every successful Open. Each pool worker opens one reader
// per input, so the count bounds the number of concurrent handles.
opens int
}
// openCount returns how many readers have been opened so far.
func (f *fakeIO) openCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return f.opens
}
type fakeRaster struct {
@@ -96,6 +107,7 @@ func (f *fakeIO) Open(_ context.Context, path string) (raster.Reader, error) {
if !ok {
return nil, fmt.Errorf("fake: %s not found", path)
}
f.opens++
return &fakeReader{f: f, path: path, r: r}, nil
}

View File

@@ -34,7 +34,9 @@ func kernelZIntegrated(in [][]float64, out [][]int16, _ Params) {
out[0][i] = NoData
continue
}
score := float32(0.25)*float32(z1) + float32(0.25)*float32(z2) + float32(0.25)*float32(z3) + float32(0.25)*float32(z4)
// explicit conversions forbid FMA fusion so results match numpy's per-op rounding
score := float32(float32(0.25)*float32(z1)) + float32(float32(0.25)*float32(z2)) +
float32(float32(0.25)*float32(z3)) + float32(float32(0.25)*float32(z4))
out[0][i] = classify(score)
}
}
@@ -49,14 +51,15 @@ func kernelUIntegrated(in [][]float64, out [][]int16, _ Params) {
out[0][i] = NoData
continue
}
num := float32(0.25)*float32(water) + float32(0.25)*float32(soil)
// explicit conversions forbid FMA fusion so results match numpy's per-op rounding
num := float32(float32(0.25)*float32(water)) + float32(float32(0.25)*float32(soil))
den := float32(0.5)
if cropOK {
num += float32(0.25) * float32(crop)
num += float32(float32(0.25) * float32(crop))
den += 0.25
}
if vegOK {
num += float32(0.25) * float32(veg)
num += float32(float32(0.25) * float32(veg))
den += 0.25
}
out[0][i] = classify(num / den)