infureal/gis
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(landscape): guard redelivery, detach terminal writes, bound workers

Browse Source 
F2: Execute had no status guard, so a redelivered or stale message reran a
finished run. Runs that are neither queued nor running are logged and acked
without work; "running" still executes so a worker that crashed mid-run
recovers.

F4: failRun, finishNodeFailed and MarkRunFailed wrote the terminal status on
the caller's context, which is exactly the context that is already cancelled
when the run failed because of a shutdown or a sibling node — the write was
refused and the run stayed stuck in "running". They now write on a
context.WithoutCancel copy with a 10s timeout; the consumer's best-effort
MarkRunFailed does the same.

F5: peak memory scaled with the host's core count. LP_MAX_WORKERS (default 8)
is now the total window-worker budget of a run and LP_MAX_NODES (default 2) the
node fan-out; each node gets max(1, workers/nodes) through the new
landscape.ComputeWithWorkers. Compute and service.ComputeFunc keep their
signatures: Compute delegates with MaxWorkersPerNode, and a legacy ComputeFunc
handed to NewLandscapeService is wrapped.

F6: the stage loop used a plain errgroup, so a failing node let its siblings
run to completion. It now uses errgroup.WithContext and passes the group
context into runNode, cancelling siblings mid-compute.

F7: wrap every float32 product in an explicit conversion so the compiler
cannot fuse it into an FMA, keeping the kernels bit-identical to numpy's
per-op rounding.

F8: README's builder package list now matches the Dockerfile (adds g++), and
the Landscape section documents the memory model and the two new knobs.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Bakhtiyar Issakhmetov
2026-09-05 14:44:34 +05:00
parent bcda76003c
commit 4051506b95
12 changed files with 563 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
internal/service/landscape.go
View File

@@ -11,7 +11,6 @@ import (
"log/slog"
"os"
"path/filepath"
"runtime"
"sort"
"sync"
"time"
@@ -29,6 +28,47 @@ import (
// mirrors landscape.Compute so tests can substitute a stub.
type ComputeFunc func(ctx context.Context, rio raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params) (map[string]string, error)
// ComputeWithWorkersFunc is ComputeFunc plus an explicit per-node worker cap.
// It mirrors landscape.ComputeWithWorkers.
type ComputeWithWorkersFunc func(ctx context.Context, rio raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params, workers int) (map[string]string, error)
// Default bounds on a run's parallelism, mirroring config.LandscapeConfig.
const (
// DefaultMaxWorkers is the total window-worker budget of one run.
DefaultMaxWorkers = 8
// DefaultMaxConcurrentNodes is how many nodes of a stage run at a time.
DefaultMaxConcurrentNodes = 2
)
// terminalWriteTimeout bounds a terminal status write made on a context that
// has already been cancelled (see terminalCtx).
const terminalWriteTimeout = 10 * time.Second
// LandscapeOptions bounds a run's parallelism, and with it the worker's peak
// memory: MaxConcurrentNodes nodes run at a time, and they share the
// MaxWorkers window-worker budget. Zero or negative values fall back to the
// defaults.
type LandscapeOptions struct {
MaxWorkers int
MaxConcurrentNodes int
}
func (o LandscapeOptions) withDefaults() LandscapeOptions {
if o.MaxWorkers < 1 {
o.MaxWorkers = DefaultMaxWorkers
}
if o.MaxConcurrentNodes < 1 {
o.MaxConcurrentNodes = DefaultMaxConcurrentNodes
}
return o
}
// workersPerNode splits the total worker budget over the concurrently running
// nodes, so nodes x workers stays at MaxWorkers. Always at least 1.
func (o LandscapeOptions) workersPerNode() int {
return max(1, o.MaxWorkers/o.MaxConcurrentNodes)
}
// LandscapeRepository is the persistence behaviour LandscapeService needs.
type LandscapeRepository interface {
CreateRun(ctx context.Context, run domain.LPRun) (domain.LPRun, error)
@@ -115,15 +155,24 @@ type LandscapeService struct {
store ObjectStore
jobs LandscapeJobEnqueuer
rio raster.IO
compute ComputeFunc
log *slog.Logger
// compute is the legacy worker-count-agnostic entry point, kept so an
// existing ComputeFunc still wires up; runNode always calls
// computeWithWorkers, which wraps compute when one was provided.
compute ComputeFunc
computeWithWorkers ComputeWithWorkersFunc
opts LandscapeOptions
log *slog.Logger
}
// NewLandscapeService wires the run repository, the dataset and group
// repositories (input resolution), the category reader (the "landscape"
// category every result is filed under), the dataset publisher (result
// datasets and slot uploads), the object store (input downloads), the job
// enqueuer, and the raster IO / compute function the graph runs on.
// enqueuer, and the raster IO / compute function the graph runs on. Pass a nil
// compute (the production wiring) to run landscape.ComputeWithWorkers with the
// per-node share of opts.MaxWorkers; a non-nil ComputeFunc is honoured for
// backward compatibility but never sees the worker cap. opts bounds the run's
// parallelism; zero fields fall back to the package defaults.
func NewLandscapeService(
runs LandscapeRepository,
datasets DatasetRepository,
@@ -135,18 +184,29 @@ func NewLandscapeService(
rio raster.IO,
compute ComputeFunc,
log *slog.Logger,
opts LandscapeOptions,
) *LandscapeService {
// Default to the workers-aware entry point; wrap a supplied legacy
// ComputeFunc (which has no worker parameter) when one is given.
withWorkers := ComputeWithWorkersFunc(landscape.ComputeWithWorkers)
if compute != nil {
withWorkers = func(ctx context.Context, rio raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params, _ int) (map[string]string, error) {
return compute(ctx, rio, node, inputs, outDir, p)
}
}
return &LandscapeService{
runs: runs,
datasets: datasets,
groups: groups,
categories: categories,
publisher: publisher,
store: store,
jobs: jobs,
rio: rio,
compute: compute,
log: log,
runs: runs,
datasets: datasets,
groups: groups,
categories: categories,
publisher: publisher,
store: store,
jobs: jobs,
rio: rio,
compute: compute,
computeWithWorkers: withWorkers,
opts: opts.withDefaults(),
log: log,
}
}
@@ -344,7 +404,9 @@ func (s *LandscapeService) Results(ctx context.Context) ([]ResultInfo, error) {
// MarkRunFailed records a terminal failure on a run so it does not stay stuck
// in an in-progress status.
func (s *LandscapeService) MarkRunFailed(ctx context.Context, runID uuid.UUID, reason string) error {
return s.runs.SetRunStatus(ctx, runID, domain.LPRunFailed, &reason)
wctx, cancel := terminalCtx(ctx)
defer cancel()
return s.runs.SetRunStatus(wctx, runID, domain.LPRunFailed, &reason)
}
// Execute runs the whole graph for one run: stages in order, nodes within a
@@ -355,6 +417,13 @@ func (s *LandscapeService) Execute(ctx context.Context, runID uuid.UUID) error {
if err != nil {
return err
}
// A redelivered or stale message must not rerun a finished run. "running"
// is allowed through so a run whose worker crashed mid-flight still
// recovers on redelivery.
if run.Status != domain.LPRunQueued && run.Status != domain.LPRunRunning {
s.log.Info("landscape run already terminal, ignoring redelivery", "run_id", runID, "status", run.Status)
return nil
}
var params landscape.Params
if len(run.Params) > 0 {
@@ -382,15 +451,18 @@ func (s *LandscapeService) Execute(ctx context.Context, runID uuid.UUID) error {
state := &runState{produced: map[string]uuid.UUID{}}
for _, stage := range landscape.Stages() {
var g errgroup.Group
// Each node's Compute spins its own worker pool (up to
// min(GOMAXPROCS, windows, 8) workers, each holding a window buffer per
// input band), so peak memory is the product of the two limits. Bound
// the node fan-out here so that product stays sane: without a limit all
// four stage-0 nodes would run at once and multiply Compute's own cap.
g.SetLimit(maxConcurrentNodes())
// WithContext so the first failing node cancels its siblings: Compute
// honours cancellation, so a doomed stage stops reading rasters
// instead of running to completion. Terminal status writes use
// terminalCtx, so they still land on the cancelled context.
g, gctx := errgroup.WithContext(ctx)
// Each node's Compute spins its own worker pool (each worker holding a
// window buffer per input band), so peak memory is the product of the
// two limits. Bound the node fan-out here and split the worker budget
// over it so that product stays at opts.MaxWorkers.
g.SetLimit(s.opts.MaxConcurrentNodes)
for _, node := range stage {
g.Go(func() error { return s.runNode(ctx, run, node, params, cat, state) })
g.Go(func() error { return s.runNode(gctx, run, node, params, cat, state) })
}
// Wait for every node of the stage: each one records its own outcome,
// and a sibling's failure must not leave a node half-recorded.
@@ -402,10 +474,20 @@ func (s *LandscapeService) Execute(ctx context.Context, runID uuid.UUID) error {
return s.runs.SetRunStatus(ctx, runID, domain.LPRunSucceeded, nil)
}
// terminalCtx derives a bounded context that outlives the cancellation of ctx.
// Terminal status writes must happen even when the run failed *because* the
// context was cancelled (shutdown, a sibling node failing the stage); on the
// original context they would be refused and leave the run stuck in "running".
func terminalCtx(ctx context.Context) (context.Context, context.CancelFunc) {
return context.WithTimeout(context.WithoutCancel(ctx), terminalWriteTimeout)
}
// failRun marks the run failed with cause's message and returns cause.
func (s *LandscapeService) failRun(ctx context.Context, runID uuid.UUID, cause error) error {
msg := cause.Error()
if err := s.runs.SetRunStatus(ctx, runID, domain.LPRunFailed, &msg); err != nil {
wctx, cancel := terminalCtx(ctx)
defer cancel()
if err := s.runs.SetRunStatus(wctx, runID, domain.LPRunFailed, &msg); err != nil {
s.log.Error("mark landscape run failed", "run_id", runID, "error", err)
}
return cause
@@ -473,7 +555,9 @@ func (s *LandscapeService) runNode(ctx context.Context, run domain.LPRun, node l
// finishNodeFailed records the failure on the node and returns the cause.
func (s *LandscapeService) finishNodeFailed(ctx context.Context, runID uuid.UUID, node landscape.Node, cause error) error {
msg := cause.Error()
if err := s.runs.SetNodeFinished(ctx, runID, node.Code, domain.LPNodeFailed, nil, &msg); err != nil {
wctx, cancel := terminalCtx(ctx)
defer cancel()
if err := s.runs.SetNodeFinished(wctx, runID, node.Code, domain.LPNodeFailed, nil, &msg); err != nil {
s.log.Error("mark landscape node failed", "run_id", runID, "node", node.Code, "error", err)
}
return cause
@@ -583,7 +667,7 @@ func (s *LandscapeService) computeNode(
paths[group] = path
}
outPaths, err := s.compute(ctx, s.rio, node, paths, dir, params)
outPaths, err := s.computeWithWorkers(ctx, s.rio, node, paths, dir, params, s.opts.workersPerNode())
if err != nil {
return nil, err
}
@@ -693,21 +777,6 @@ func knownGroup(code string) bool {
return ok
}
// maxConcurrentNodes caps how many graph nodes of one stage run at the same
// time. Compute already parallelises within a node, so half the CPUs (at most
// 4, the widest stage) is enough to keep the machine busy without letting the
// two levels of parallelism multiply into an out-of-memory.
func maxConcurrentNodes() int {
n := runtime.GOMAXPROCS(0) / 2
if n < 1 {
n = 1
}
if n > 4 {
n = 4
}
return n
}
// maxStage returns the highest stage number in the graph.
func maxStage() int {
highest := 0