Bakhtiyar Issakhmetov
4051506b95
F2: Execute had no status guard, so a redelivered or stale message reran a finished run. Runs that are neither queued nor running are logged and acked without work; "running" still executes so a worker that crashed mid-run recovers. F4: failRun, finishNodeFailed and MarkRunFailed wrote the terminal status on the caller's context, which is exactly the context that is already cancelled when the run failed because of a shutdown or a sibling node — the write was refused and the run stayed stuck in "running". They now write on a context.WithoutCancel copy with a 10s timeout; the consumer's best-effort MarkRunFailed does the same. F5: peak memory scaled with the host's core count. LP_MAX_WORKERS (default 8) is now the total window-worker budget of a run and LP_MAX_NODES (default 2) the node fan-out; each node gets max(1, workers/nodes) through the new landscape.ComputeWithWorkers. Compute and service.ComputeFunc keep their signatures: Compute delegates with MaxWorkersPerNode, and a legacy ComputeFunc handed to NewLandscapeService is wrapped. F6: the stage loop used a plain errgroup, so a failing node let its siblings run to completion. It now uses errgroup.WithContext and passes the group context into runNode, cancelling siblings mid-compute. F7: wrap every float32 product in an explicit conversion so the compiler cannot fuse it into an FMA, keeping the kernels bit-identical to numpy's per-op rounding. F8: README's builder package list now matches the Dockerfile (adds g++), and the Landscape section documents the memory model and the two new knobs. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
87 lines
3.1 KiB
Go
87 lines
3.1 KiB
Go
// Package config loads application configuration from the environment.
|
|
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/caarlos0/env/v11"
|
|
"github.com/joho/godotenv"
|
|
)
|
|
|
|
// Config holds all configuration for the application. Values are read from
|
|
// environment variables; a local .env file (if present) is loaded first.
|
|
type Config struct {
|
|
HTTP HTTPConfig
|
|
DB DBConfig
|
|
S3 S3Config
|
|
RabbitMQ RabbitMQConfig
|
|
Landscape LandscapeConfig
|
|
}
|
|
|
|
// HTTPConfig configures the HTTP server.
|
|
type HTTPConfig struct {
|
|
Port int `env:"PORT" envDefault:"8080"`
|
|
ReadHeaderTimeout time.Duration `env:"HTTP_READ_HEADER_TIMEOUT" envDefault:"5s"`
|
|
ReadTimeout time.Duration `env:"HTTP_READ_TIMEOUT" envDefault:"120s"`
|
|
WriteTimeout time.Duration `env:"HTTP_WRITE_TIMEOUT" envDefault:"120s"`
|
|
IdleTimeout time.Duration `env:"HTTP_IDLE_TIMEOUT" envDefault:"60s"`
|
|
ShutdownTimeout time.Duration `env:"HTTP_SHUTDOWN_TIMEOUT" envDefault:"10s"`
|
|
}
|
|
|
|
// Addr returns the listen address for the HTTP server.
|
|
func (c HTTPConfig) Addr() string {
|
|
return fmt.Sprintf(":%d", c.Port)
|
|
}
|
|
|
|
// DBConfig configures the Postgres connection.
|
|
type DBConfig struct {
|
|
URL string `env:"DB_URL,required"`
|
|
// Schema is the Postgres schema migrations operate on. It is used by
|
|
// `migrate fresh` to know which schema to drop and recreate; in production
|
|
// this may be something other than "public".
|
|
Schema string `env:"DB_SCHEMA" envDefault:"public"`
|
|
}
|
|
|
|
// S3Config configures the S3/MinIO object store.
|
|
type S3Config struct {
|
|
Endpoint string `env:"S3_ENDPOINT,required"`
|
|
AccessKey string `env:"S3_ACCESS_KEY,required"`
|
|
SecretKey string `env:"S3_SECRET_KEY,required"`
|
|
Bucket string `env:"S3_BUCKET" envDefault:"geofiles"`
|
|
UseSSL bool `env:"S3_USE_SSL" envDefault:"false"`
|
|
}
|
|
|
|
// RabbitMQConfig configures the RabbitMQ connection and example topology.
|
|
type RabbitMQConfig struct {
|
|
URL string `env:"RABBITMQ_URL,required"`
|
|
Exchange string `env:"RABBITMQ_EXCHANGE" envDefault:"gis.events"`
|
|
Queue string `env:"RABBITMQ_QUEUE" envDefault:"gis.events.example"`
|
|
}
|
|
|
|
// LandscapeConfig bounds the Landscape Program's parallelism, and with it the
|
|
// worker's peak memory: a run holds up to MaxConcurrentNodes nodes in flight,
|
|
// each running a pool of MaxWorkers/MaxConcurrentNodes goroutines that own one
|
|
// window buffer per input and output. The defaults (8 total workers spread
|
|
// over 2 nodes) keep that product bounded regardless of how many cores the
|
|
// host has.
|
|
type LandscapeConfig struct {
|
|
// MaxWorkers is the total window-worker budget shared by all nodes of a run.
|
|
MaxWorkers int `env:"LP_MAX_WORKERS" envDefault:"8"`
|
|
// MaxConcurrentNodes is how many graph nodes of one stage run at a time.
|
|
MaxConcurrentNodes int `env:"LP_MAX_NODES" envDefault:"2"`
|
|
}
|
|
|
|
// Load reads configuration from the environment, loading an optional .env file
|
|
// from the current working directory first.
|
|
func Load() (*Config, error) {
|
|
// A missing .env file is not an error: in production we rely on real env vars.
|
|
_ = godotenv.Load()
|
|
|
|
cfg := &Config{}
|
|
if err := env.Parse(cfg); err != nil {
|
|
return nil, fmt.Errorf("parse config: %w", err)
|
|
}
|
|
return cfg, nil
|
|
}
|