fix(landscape): guard redelivery, detach terminal writes, bound workers

F2: Execute had no status guard, so a redelivered or stale message reran a finished run. Runs that are neither queued nor running are logged and acked without work; "running" still executes so a worker that crashed mid-run recovers. F4: failRun, finishNodeFailed and MarkRunFailed wrote the terminal status on the caller's context, which is exactly the context that is already cancelled when the run failed because of a shutdown or a sibling node — the write was refused and the run stayed stuck in "running". They now write on a context.WithoutCancel copy with a 10s timeout; the consumer's best-effort MarkRunFailed does the same. F5: peak memory scaled with the host's core count. LP_MAX_WORKERS (default 8) is now the total window-worker budget of a run and LP_MAX_NODES (default 2) the node fan-out; each node gets max(1, workers/nodes) through the new landscape.ComputeWithWorkers. Compute and service.ComputeFunc keep their signatures: Compute delegates with MaxWorkersPerNode, and a legacy ComputeFunc handed to NewLandscapeService is wrapped. F6: the stage loop used a plain errgroup, so a failing node let its siblings run to completion. It now uses errgroup.WithContext and passes the group context into runNode, cancelling siblings mid-compute. F7: wrap every float32 product in an explicit conversion so the compiler cannot fuse it into an FMA, keeping the kernels bit-identical to numpy's per-op rounding. F8: README's builder package list now matches the Dockerfile (adds g++), and the Landscape section documents the memory model and the two new knobs. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-05 14:44:34 +05:00
parent bcda76003c
commit 4051506b95
12 changed files with 563 additions and 69 deletions
--- a/internal/service/landscape.go
+++ b/internal/service/landscape.go
@@ -11,7 +11,6 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
-	"runtime"
 	"sort"
 	"sync"
 	"time"
@@ -29,6 +28,47 @@ import (
 // mirrors landscape.Compute so tests can substitute a stub.
 type ComputeFunc func(ctx context.Context, rio raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params) (map[string]string, error)

+// ComputeWithWorkersFunc is ComputeFunc plus an explicit per-node worker cap.
+// It mirrors landscape.ComputeWithWorkers.
+type ComputeWithWorkersFunc func(ctx context.Context, rio raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params, workers int) (map[string]string, error)
+
+// Default bounds on a run's parallelism, mirroring config.LandscapeConfig.
+const (
+	// DefaultMaxWorkers is the total window-worker budget of one run.
+	DefaultMaxWorkers = 8
+	// DefaultMaxConcurrentNodes is how many nodes of a stage run at a time.
+	DefaultMaxConcurrentNodes = 2
+)
+
+// terminalWriteTimeout bounds a terminal status write made on a context that
+// has already been cancelled (see terminalCtx).
+const terminalWriteTimeout = 10 * time.Second
+
+// LandscapeOptions bounds a run's parallelism, and with it the worker's peak
+// memory: MaxConcurrentNodes nodes run at a time, and they share the
+// MaxWorkers window-worker budget. Zero or negative values fall back to the
+// defaults.
+type LandscapeOptions struct {
+	MaxWorkers         int
+	MaxConcurrentNodes int
+}
+
+func (o LandscapeOptions) withDefaults() LandscapeOptions {
+	if o.MaxWorkers < 1 {
+		o.MaxWorkers = DefaultMaxWorkers
+	}
+	if o.MaxConcurrentNodes < 1 {
+		o.MaxConcurrentNodes = DefaultMaxConcurrentNodes
+	}
+	return o
+}
+
+// workersPerNode splits the total worker budget over the concurrently running
+// nodes, so nodes x workers stays at MaxWorkers. Always at least 1.
+func (o LandscapeOptions) workersPerNode() int {
+	return max(1, o.MaxWorkers/o.MaxConcurrentNodes)
+}
+
 // LandscapeRepository is the persistence behaviour LandscapeService needs.
 type LandscapeRepository interface {
 	CreateRun(ctx context.Context, run domain.LPRun) (domain.LPRun, error)
@@ -115,15 +155,24 @@ type LandscapeService struct {
 	store      ObjectStore
 	jobs       LandscapeJobEnqueuer
 	rio        raster.IO
-	compute    ComputeFunc
-	log        *slog.Logger
+	// compute is the legacy worker-count-agnostic entry point, kept so an
+	// existing ComputeFunc still wires up; runNode always calls
+	// computeWithWorkers, which wraps compute when one was provided.
+	compute            ComputeFunc
+	computeWithWorkers ComputeWithWorkersFunc
+	opts               LandscapeOptions
+	log                *slog.Logger
 }

 // NewLandscapeService wires the run repository, the dataset and group
 // repositories (input resolution), the category reader (the "landscape"
 // category every result is filed under), the dataset publisher (result
 // datasets and slot uploads), the object store (input downloads), the job
-// enqueuer, and the raster IO / compute function the graph runs on.
+// enqueuer, and the raster IO / compute function the graph runs on. Pass a nil
+// compute (the production wiring) to run landscape.ComputeWithWorkers with the
+// per-node share of opts.MaxWorkers; a non-nil ComputeFunc is honoured for
+// backward compatibility but never sees the worker cap. opts bounds the run's
+// parallelism; zero fields fall back to the package defaults.
 func NewLandscapeService(
 	runs LandscapeRepository,
 	datasets DatasetRepository,
@@ -135,18 +184,29 @@ func NewLandscapeService(
 	rio raster.IO,
 	compute ComputeFunc,
 	log *slog.Logger,
+	opts LandscapeOptions,
 ) *LandscapeService {
+	// Default to the workers-aware entry point; wrap a supplied legacy
+	// ComputeFunc (which has no worker parameter) when one is given.
+	withWorkers := ComputeWithWorkersFunc(landscape.ComputeWithWorkers)
+	if compute != nil {
+		withWorkers = func(ctx context.Context, rio raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params, _ int) (map[string]string, error) {
+			return compute(ctx, rio, node, inputs, outDir, p)
+		}
+	}
 	return &LandscapeService{
-		runs:       runs,
-		datasets:   datasets,
-		groups:     groups,
-		categories: categories,
-		publisher:  publisher,
-		store:      store,
-		jobs:       jobs,
-		rio:        rio,
-		compute:    compute,
-		log:        log,
+		runs:               runs,
+		datasets:           datasets,
+		groups:             groups,
+		categories:         categories,
+		publisher:          publisher,
+		store:              store,
+		jobs:               jobs,
+		rio:                rio,
+		compute:            compute,
+		computeWithWorkers: withWorkers,
+		opts:               opts.withDefaults(),
+		log:                log,
 	}
 }

@@ -344,7 +404,9 @@ func (s *LandscapeService) Results(ctx context.Context) ([]ResultInfo, error) {
 // MarkRunFailed records a terminal failure on a run so it does not stay stuck
 // in an in-progress status.
 func (s *LandscapeService) MarkRunFailed(ctx context.Context, runID uuid.UUID, reason string) error {
-	return s.runs.SetRunStatus(ctx, runID, domain.LPRunFailed, &reason)
+	wctx, cancel := terminalCtx(ctx)
+	defer cancel()
+	return s.runs.SetRunStatus(wctx, runID, domain.LPRunFailed, &reason)
 }

 // Execute runs the whole graph for one run: stages in order, nodes within a
@@ -355,6 +417,13 @@ func (s *LandscapeService) Execute(ctx context.Context, runID uuid.UUID) error {
 	if err != nil {
 		return err
 	}
+	// A redelivered or stale message must not rerun a finished run. "running"
+	// is allowed through so a run whose worker crashed mid-flight still
+	// recovers on redelivery.
+	if run.Status != domain.LPRunQueued && run.Status != domain.LPRunRunning {
+		s.log.Info("landscape run already terminal, ignoring redelivery", "run_id", runID, "status", run.Status)
+		return nil
+	}

 	var params landscape.Params
 	if len(run.Params) > 0 {
@@ -382,15 +451,18 @@ func (s *LandscapeService) Execute(ctx context.Context, runID uuid.UUID) error {

 	state := &runState{produced: map[string]uuid.UUID{}}
 	for _, stage := range landscape.Stages() {
-		var g errgroup.Group
-		// Each node's Compute spins its own worker pool (up to
-		// min(GOMAXPROCS, windows, 8) workers, each holding a window buffer per
-		// input band), so peak memory is the product of the two limits. Bound
-		// the node fan-out here so that product stays sane: without a limit all
-		// four stage-0 nodes would run at once and multiply Compute's own cap.
-		g.SetLimit(maxConcurrentNodes())
+		// WithContext so the first failing node cancels its siblings: Compute
+		// honours cancellation, so a doomed stage stops reading rasters
+		// instead of running to completion. Terminal status writes use
+		// terminalCtx, so they still land on the cancelled context.
+		g, gctx := errgroup.WithContext(ctx)
+		// Each node's Compute spins its own worker pool (each worker holding a
+		// window buffer per input band), so peak memory is the product of the
+		// two limits. Bound the node fan-out here and split the worker budget
+		// over it so that product stays at opts.MaxWorkers.
+		g.SetLimit(s.opts.MaxConcurrentNodes)
 		for _, node := range stage {
-			g.Go(func() error { return s.runNode(ctx, run, node, params, cat, state) })
+			g.Go(func() error { return s.runNode(gctx, run, node, params, cat, state) })
 		}
 		// Wait for every node of the stage: each one records its own outcome,
 		// and a sibling's failure must not leave a node half-recorded.
@@ -402,10 +474,20 @@ func (s *LandscapeService) Execute(ctx context.Context, runID uuid.UUID) error {
 	return s.runs.SetRunStatus(ctx, runID, domain.LPRunSucceeded, nil)
 }

+// terminalCtx derives a bounded context that outlives the cancellation of ctx.
+// Terminal status writes must happen even when the run failed *because* the
+// context was cancelled (shutdown, a sibling node failing the stage); on the
+// original context they would be refused and leave the run stuck in "running".
+func terminalCtx(ctx context.Context) (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.WithoutCancel(ctx), terminalWriteTimeout)
+}
+
 // failRun marks the run failed with cause's message and returns cause.
 func (s *LandscapeService) failRun(ctx context.Context, runID uuid.UUID, cause error) error {
 	msg := cause.Error()
-	if err := s.runs.SetRunStatus(ctx, runID, domain.LPRunFailed, &msg); err != nil {
+	wctx, cancel := terminalCtx(ctx)
+	defer cancel()
+	if err := s.runs.SetRunStatus(wctx, runID, domain.LPRunFailed, &msg); err != nil {
 		s.log.Error("mark landscape run failed", "run_id", runID, "error", err)
 	}
 	return cause
@@ -473,7 +555,9 @@ func (s *LandscapeService) runNode(ctx context.Context, run domain.LPRun, node l
 // finishNodeFailed records the failure on the node and returns the cause.
 func (s *LandscapeService) finishNodeFailed(ctx context.Context, runID uuid.UUID, node landscape.Node, cause error) error {
 	msg := cause.Error()
-	if err := s.runs.SetNodeFinished(ctx, runID, node.Code, domain.LPNodeFailed, nil, &msg); err != nil {
+	wctx, cancel := terminalCtx(ctx)
+	defer cancel()
+	if err := s.runs.SetNodeFinished(wctx, runID, node.Code, domain.LPNodeFailed, nil, &msg); err != nil {
 		s.log.Error("mark landscape node failed", "run_id", runID, "node", node.Code, "error", err)
 	}
 	return cause
@@ -583,7 +667,7 @@ func (s *LandscapeService) computeNode(
 		paths[group] = path
 	}

-	outPaths, err := s.compute(ctx, s.rio, node, paths, dir, params)
+	outPaths, err := s.computeWithWorkers(ctx, s.rio, node, paths, dir, params, s.opts.workersPerNode())
 	if err != nil {
 		return nil, err
 	}
@@ -693,21 +777,6 @@ func knownGroup(code string) bool {
 	return ok
 }

-// maxConcurrentNodes caps how many graph nodes of one stage run at the same
-// time. Compute already parallelises within a node, so half the CPUs (at most
-// 4, the widest stage) is enough to keep the machine busy without letting the
-// two levels of parallelism multiply into an out-of-memory.
-func maxConcurrentNodes() int {
-	n := runtime.GOMAXPROCS(0) / 2
-	if n < 1 {
-		n = 1
-	}
-	if n > 4 {
-		n = 4
-	}
-	return n
-}
-
 // maxStage returns the highest stage number in the graph.
 func maxStage() int {
 	highest := 0
--- a/internal/service/landscape_test.go
+++ b/internal/service/landscape_test.go
@@ -65,12 +65,63 @@ func (r *lockedDatasetRepo) LatestInGroup(ctx context.Context, groupID uuid.UUID
 }

 // stubLPRepo is an in-memory LandscapeRepository for tests.
+// lpStatusWrite records one SetRunStatus call and whether the context it was
+// handed was still live.
+type lpStatusWrite struct {
+	status domain.LPRunStatus
+	ctxErr error
+}
+
+// lpNodeWrite records one SetNodeFinished call the same way.
+type lpNodeWrite struct {
+	code   string
+	status domain.LPNodeStatus
+	ctxErr error
+}
+
 type stubLPRepo struct {
 	mu           sync.Mutex
 	runs         map[uuid.UUID]*domain.LPRun
 	order        []uuid.UUID
 	reuseLookups []string
 	clock        int64
+	statusWrites []lpStatusWrite
+	nodeWrites   []lpNodeWrite
+}
+
+// statusesWritten returns the run statuses written so far, in order.
+func (r *stubLPRepo) statusesWritten() []domain.LPRunStatus {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	out := make([]domain.LPRunStatus, 0, len(r.statusWrites))
+	for _, w := range r.statusWrites {
+		out = append(out, w.status)
+	}
+	return out
+}
+
+// statusWrite returns the last write recording the given status.
+func (r *stubLPRepo) statusWrite(status domain.LPRunStatus) (lpStatusWrite, bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for i := len(r.statusWrites) - 1; i >= 0; i-- {
+		if r.statusWrites[i].status == status {
+			return r.statusWrites[i], true
+		}
+	}
+	return lpStatusWrite{}, false
+}
+
+// nodeWrite returns the last SetNodeFinished write recording the given status.
+func (r *stubLPRepo) nodeWrite(status domain.LPNodeStatus) (lpNodeWrite, bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for i := len(r.nodeWrites) - 1; i >= 0; i-- {
+		if r.nodeWrites[i].status == status {
+			return r.nodeWrites[i], true
+		}
+	}
+	return lpNodeWrite{}, false
 }

 func newStubLPRepo() *stubLPRepo {
@@ -170,9 +221,10 @@ func (r *stubLPRepo) CountRuns(_ context.Context) (int, error) {
 	return len(r.runs), nil
 }

-func (r *stubLPRepo) SetRunStatus(_ context.Context, id uuid.UUID, status domain.LPRunStatus, errMsg *string) error {
+func (r *stubLPRepo) SetRunStatus(ctx context.Context, id uuid.UUID, status domain.LPRunStatus, errMsg *string) error {
 	r.mu.Lock()
 	defer r.mu.Unlock()
+	r.statusWrites = append(r.statusWrites, lpStatusWrite{status: status, ctxErr: ctx.Err()})
 	run, ok := r.runs[id]
 	if !ok {
 		return domain.ErrNotFound
@@ -219,9 +271,10 @@ func (r *stubLPRepo) SetNodeStarted(_ context.Context, runID uuid.UUID, code, fi
 	return nil
 }

-func (r *stubLPRepo) SetNodeFinished(_ context.Context, runID uuid.UUID, code string, status domain.LPNodeStatus, outputs []domain.LPDatasetRef, errMsg *string) error {
+func (r *stubLPRepo) SetNodeFinished(ctx context.Context, runID uuid.UUID, code string, status domain.LPNodeStatus, outputs []domain.LPDatasetRef, errMsg *string) error {
 	r.mu.Lock()
 	defer r.mu.Unlock()
+	r.nodeWrites = append(r.nodeWrites, lpNodeWrite{code: code, status: status, ctxErr: ctx.Err()})
 	n, err := r.node(runID, code)
 	if err != nil {
 		return err
@@ -354,8 +407,11 @@ func (j *stubLPJobs) EnqueueRun(_ context.Context, runID uuid.UUID) error {

 // stubComputeCall records one invocation of the compute stub.
 type stubComputeCall struct {
-	node   string
-	inputs map[string]string
+	node string
+	// workers is the per-node worker cap the service passed, or -1 when the
+	// call came through the legacy worker-agnostic ComputeFunc.
+	workers int
+	inputs  map[string]string
 	// bodies holds each input file's content at call time; seedDataset stores
 	// each object under its own storage key as its body, so a test can tell
 	// which dataset a node actually downloaded.
@@ -369,11 +425,47 @@ type stubCompute struct {
 	mu    sync.Mutex
 	calls []stubComputeCall
 	errs  map[string]error
+	// waitForCancel names nodes whose compute blocks until its context is
+	// cancelled (or the deadline below elapses), so a test can observe whether
+	// a sibling's failure cancelled them.
+	waitForCancel map[string]bool
+	sawCancel     map[string]bool
 }

-func newStubCompute() *stubCompute { return &stubCompute{errs: map[string]error{}} }
+func newStubCompute() *stubCompute {
+	return &stubCompute{errs: map[string]error{}, waitForCancel: map[string]bool{}, sawCancel: map[string]bool{}}
+}
+
+// fn is a legacy service.ComputeFunc: it sees no worker cap.
+func (c *stubCompute) fn(ctx context.Context, rio raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params) (map[string]string, error) {
+	return c.fnWithWorkers(ctx, rio, node, inputs, outDir, p, -1)
+}
+
+// observedCancel reports whether the node's compute saw a cancelled context.
+func (c *stubCompute) observedCancel(code string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.sawCancel[code]
+}
+
+func (c *stubCompute) fnWithWorkers(ctx context.Context, _ raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params, workers int) (map[string]string, error) {
+	c.mu.Lock()
+	wait := c.waitForCancel[node.Code]
+	c.mu.Unlock()
+	if wait {
+		select {
+		case <-ctx.Done():
+		case <-time.After(5 * time.Second):
+		}
+		c.mu.Lock()
+		c.sawCancel[node.Code] = ctx.Err() != nil
+		c.mu.Unlock()
+	}
+	// The real Compute honours cancellation, so the stub must too.
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}

-func (c *stubCompute) fn(_ context.Context, _ raster.IO, node landscape.Node, inputs map[string]string, outDir string, p landscape.Params) (map[string]string, error) {
 	in := map[string]string{}
 	bodies := map[string]string{}
 	for k, v := range inputs {
@@ -386,7 +478,7 @@ func (c *stubCompute) fn(_ context.Context, _ raster.IO, node landscape.Node, in
 	}

 	c.mu.Lock()
-	c.calls = append(c.calls, stubComputeCall{node: node.Code, inputs: in, bodies: bodies, params: p})
+	c.calls = append(c.calls, stubComputeCall{node: node.Code, workers: workers, inputs: in, bodies: bodies, params: p})
 	err := c.errs[node.Code]
 	c.mu.Unlock()
 	if err != nil {
@@ -453,6 +545,12 @@ type lpFixture struct {
 }

 func newLPFixture(t *testing.T) *lpFixture {
+	t.Helper()
+	return newLPFixtureWith(t, LandscapeOptions{})
+}
+
+// newLPFixtureWith builds the fixture with explicit parallelism options.
+func newLPFixtureWith(t *testing.T, opts LandscapeOptions) *lpFixture {
 	t.Helper()
 	ctx := context.Background()

@@ -482,7 +580,11 @@ func newLPFixture(t *testing.T) *lpFixture {
 	comp := newStubCompute()
 	log := slog.New(slog.NewTextHandler(io.Discard, nil))

-	svc := NewLandscapeService(runs, datasets, groups, cats, pub, store, jobs, nil, comp.fn, log)
+	svc := NewLandscapeService(runs, datasets, groups, cats, pub, store, jobs, nil, nil, log, opts)
+	// Substitute the workers-aware stub so tests can see the per-node cap the
+	// service computes; NewLandscapeService's legacy-ComputeFunc wrapping is
+	// covered separately.
+	svc.computeWithWorkers = comp.fnWithWorkers
 	return &lpFixture{
 		svc: svc, runs: runs, datasets: datasets, groups: groups, cats: cats,
 		pub: pub, store: store, jobs: jobs, compute: comp, category: cat,
@@ -1216,3 +1318,203 @@ func TestLandscapeService_Execute_OverridePinsInputOverFresherDataset(t *testing
 		t.Fatalf("node 01 input for %s = %v, want the freshest %v", landscape.SlotUVeg, got, newest.ID)
 	}
 }
+
+// F2: a redelivered message for a finished run must be acked without rerunning.
+func TestLandscapeService_Execute_IgnoresTerminalRun(t *testing.T) {
+	f := newLPFixture(t)
+	ctx := context.Background()
+	f.seedAllSlots(t)
+
+	run, err := f.svc.CreateRun(ctx, RunInput{})
+	if err != nil {
+		t.Fatalf("CreateRun: %v", err)
+	}
+	if err := f.runs.SetRunStatus(ctx, run.ID, domain.LPRunSucceeded, nil); err != nil {
+		t.Fatal(err)
+	}
+	f.runs.mu.Lock()
+	f.runs.statusWrites = nil
+	f.runs.mu.Unlock()
+
+	if err := f.svc.Execute(ctx, run.ID); err != nil {
+		t.Fatalf("Execute on a terminal run must ack, got %v", err)
+	}
+	if n := f.compute.count(); n != 0 {
+		t.Fatalf("compute called %d times for a terminal run", n)
+	}
+	if got := f.runs.statusesWritten(); len(got) != 0 {
+		t.Fatalf("no status write expected, got %v", got)
+	}
+	got, err := f.svc.GetRun(ctx, run.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Status != domain.LPRunSucceeded {
+		t.Fatalf("run status changed to %s", got.Status)
+	}
+}
+
+// F2: a "running" run is still executed, so a worker that crashed mid-run
+// recovers on redelivery.
+func TestLandscapeService_Execute_RerunsRunningRun(t *testing.T) {
+	f := newLPFixture(t)
+	ctx := context.Background()
+	f.seedAllSlots(t)
+
+	run, err := f.svc.CreateRun(ctx, RunInput{})
+	if err != nil {
+		t.Fatalf("CreateRun: %v", err)
+	}
+	if err := f.runs.SetRunStatus(ctx, run.ID, domain.LPRunRunning, nil); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.svc.Execute(ctx, run.ID); err != nil {
+		t.Fatalf("Execute: %v", err)
+	}
+	if f.compute.count() == 0 {
+		t.Fatal("a running run must be executed on redelivery")
+	}
+}
+
+// F4: terminal status writes must land even when the caller's context is dead.
+func TestLandscapeService_Execute_TerminalWritesSurviveCancelledContext(t *testing.T) {
+	f := newLPFixture(t)
+	f.seedAllSlots(t)
+
+	run, err := f.svc.CreateRun(context.Background(), RunInput{})
+	if err != nil {
+		t.Fatalf("CreateRun: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := f.svc.Execute(ctx, run.ID); err == nil {
+		t.Fatal("expected Execute to fail on a cancelled context")
+	}
+
+	got, err := f.svc.GetRun(context.Background(), run.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Status != domain.LPRunFailed {
+		t.Fatalf("run status = %s, want failed", got.Status)
+	}
+	w, ok := f.runs.statusWrite(domain.LPRunFailed)
+	if !ok {
+		t.Fatal("no failed status write recorded")
+	}
+	if w.ctxErr != nil {
+		t.Fatalf("terminal run status write got a cancelled context: %v", w.ctxErr)
+	}
+	nw, ok := f.runs.nodeWrite(domain.LPNodeFailed)
+	if !ok {
+		t.Fatal("no failed node write recorded")
+	}
+	if nw.ctxErr != nil {
+		t.Fatalf("terminal node write got a cancelled context: %v", nw.ctxErr)
+	}
+}
+
+// F4: MarkRunFailed is the consumer's last resort and must not need a live ctx.
+func TestLandscapeService_MarkRunFailed_IgnoresCancelledContext(t *testing.T) {
+	f := newLPFixture(t)
+	run, err := f.svc.CreateRun(context.Background(), RunInput{})
+	if err != nil {
+		t.Fatalf("CreateRun: %v", err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := f.svc.MarkRunFailed(ctx, run.ID, "boom"); err != nil {
+		t.Fatalf("MarkRunFailed: %v", err)
+	}
+	w, ok := f.runs.statusWrite(domain.LPRunFailed)
+	if !ok {
+		t.Fatal("no failed status write recorded")
+	}
+	if w.ctxErr != nil {
+		t.Fatalf("MarkRunFailed wrote on a cancelled context: %v", w.ctxErr)
+	}
+}
+
+// F5: the per-node worker cap handed to Compute is the run's budget split over
+// the concurrently running nodes.
+func TestLandscapeService_Execute_SplitsWorkerBudgetOverNodes(t *testing.T) {
+	cases := []struct {
+		name string
+		opts LandscapeOptions
+		want int
+	}{
+		{"defaults", LandscapeOptions{}, DefaultMaxWorkers / DefaultMaxConcurrentNodes},
+		{"explicit", LandscapeOptions{MaxWorkers: 12, MaxConcurrentNodes: 3}, 4},
+		{"single node keeps the whole budget", LandscapeOptions{MaxWorkers: 6, MaxConcurrentNodes: 1}, 6},
+		{"never below one", LandscapeOptions{MaxWorkers: 1, MaxConcurrentNodes: 4}, 1},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			f := newLPFixtureWith(t, c.opts)
+			f.seedAllSlots(t)
+			got := f.runOnce(t, RunInput{})
+			if got.Status != domain.LPRunSucceeded {
+				t.Fatalf("run status = %s: %v", got.Status, got.Error)
+			}
+			f.compute.mu.Lock()
+			defer f.compute.mu.Unlock()
+			if len(f.compute.calls) == 0 {
+				t.Fatal("compute was never called")
+			}
+			for _, call := range f.compute.calls {
+				if call.workers != c.want {
+					t.Fatalf("node %s got workers=%d, want %d", call.node, call.workers, c.want)
+				}
+			}
+		})
+	}
+}
+
+// F6: a failing node cancels its siblings instead of letting them run on.
+func TestLandscapeService_Execute_FailingNodeCancelsSiblings(t *testing.T) {
+	// Four stage-0 nodes, all in flight at once; "00" fails immediately while
+	// "01" blocks waiting for its context.
+	f := newLPFixtureWith(t, LandscapeOptions{MaxWorkers: 8, MaxConcurrentNodes: 4})
+	f.seedAllSlots(t)
+	f.compute.errs["00"] = errors.New("boom")
+	f.compute.waitForCancel["01"] = true
+
+	got := f.runOnce(t, RunInput{})
+	if got.Status != domain.LPRunFailed {
+		t.Fatalf("run status = %s, want failed", got.Status)
+	}
+	if !f.compute.observedCancel("01") {
+		t.Fatal("sibling node 01 did not observe the cancellation of its context")
+	}
+	for _, n := range got.Nodes {
+		if n.NodeCode == "01" && n.Status == domain.LPNodeSucceeded {
+			t.Fatal("cancelled sibling must not be recorded as succeeded")
+		}
+	}
+}
+
+// F5: a legacy ComputeFunc still wires up through NewLandscapeService.
+func TestNewLandscapeService_WrapsLegacyComputeFunc(t *testing.T) {
+	f := newLPFixture(t)
+	f.seedAllSlots(t)
+	// Rebuild the service the production-adjacent way: a legacy ComputeFunc,
+	// no direct field substitution.
+	log := slog.New(slog.NewTextHandler(io.Discard, nil))
+	svc := NewLandscapeService(f.runs, f.datasets, f.groups, f.cats, f.pub, f.store, f.jobs, nil, f.compute.fn, log, LandscapeOptions{})
+
+	run, err := svc.CreateRun(context.Background(), RunInput{})
+	if err != nil {
+		t.Fatalf("CreateRun: %v", err)
+	}
+	if err := svc.Execute(context.Background(), run.ID); err != nil {
+		t.Fatalf("Execute: %v", err)
+	}
+	call, ok := f.compute.callFor("00")
+	if !ok {
+		t.Fatal("legacy ComputeFunc was never called")
+	}
+	if call.workers != -1 {
+		t.Fatalf("legacy ComputeFunc should not see a worker cap, got %d", call.workers)
+	}
+}