Second checkpoint

internal/parser/gpkg.go

@@ -0,0 +1,152 @@
+package parser
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"strings"
+
+	"gis/internal/domain"
+
+	_ "modernc.org/sqlite" // pure-Go SQLite driver, registered as "sqlite"
+)
+
+// withGPKG writes the GeoPackage bytes to a temp file (SQLite needs a path),
+// opens it, and runs fn with the feature table name and its attribute columns
+// (geometry column excluded).
+func withGPKG(data []byte, fn func(db *sql.DB, table string, names []string) error) error {
+	tmp, err := os.CreateTemp("", "gis-*.gpkg")
+	if err != nil {
+		return fmt.Errorf("temp file: %w", err)
+	}
+	defer os.Remove(tmp.Name())
+
+	if _, err := tmp.Write(data); err != nil {
+		tmp.Close()
+		return fmt.Errorf("write temp gpkg: %w", err)
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+
+	db, err := sql.Open("sqlite", tmp.Name())
+	if err != nil {
+		return fmt.Errorf("open gpkg: %w", err)
+	}
+	defer db.Close()
+
+	var table string
+	if err := db.QueryRow(
+		`SELECT table_name FROM gpkg_contents WHERE data_type = 'features' ORDER BY table_name LIMIT 1`,
+	).Scan(&table); err != nil {
+		return fmt.Errorf("find feature table: %w", err)
+	}
+
+	var geomColumn string
+	_ = db.QueryRow(
+		`SELECT column_name FROM gpkg_geometry_columns WHERE table_name = ?`, table,
+	).Scan(&geomColumn)
+
+	rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", quoteIdent(table)))
+	if err != nil {
+		return fmt.Errorf("read columns: %w", err)
+	}
+	defer rows.Close()
+
+	var names []string
+	for rows.Next() {
+		var (
+			cid, notnull, pk int
+			name, ctype      string
+			dflt             sql.NullString
+		)
+		if err := rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk); err != nil {
+			return err
+		}
+		if name == geomColumn {
+			continue
+		}
+		names = append(names, name)
+	}
+	if err := rows.Err(); err != nil {
+		return err
+	}
+	if len(names) == 0 {
+		return ErrNoColumns
+	}
+
+	return fn(db, table, names)
+}
+
+// gpkgColumns reads the feature table's attribute columns, with samples.
+func gpkgColumns(data []byte) ([]domain.AttributeColumn, error) {
+	var cols []domain.AttributeColumn
+	err := withGPKG(data, func(db *sql.DB, table string, names []string) error {
+		samples := gpkgScan(db, table, names, sampleRows)
+		cols = make([]domain.AttributeColumn, len(names))
+		for i, n := range names {
+			col := domain.AttributeColumn{Name: n}
+			for _, row := range samples {
+				col.Samples = append(col.Samples, row[n])
+			}
+			cols[i] = col
+		}
+		return nil
+	})
+	return cols, err
+}
+
+// gpkgRows reads every feature row as a name->value map.
+func gpkgRows(data []byte) ([]map[string]string, error) {
+	var out []map[string]string
+	err := withGPKG(data, func(db *sql.DB, table string, names []string) error {
+		out = gpkgScan(db, table, names, -1)
+		return nil
+	})
+	return out, err
+}
+
+// gpkgScan returns up to limit rows (limit < 0 means all) as name->value maps.
+func gpkgScan(db *sql.DB, table string, names []string, limit int) []map[string]string {
+	quoted := make([]string, len(names))
+	for i, n := range names {
+		quoted[i] = quoteIdent(n)
+	}
+	query := fmt.Sprintf("SELECT %s FROM %s", strings.Join(quoted, ", "), quoteIdent(table))
+	if limit >= 0 {
+		query += fmt.Sprintf(" LIMIT %d", limit)
+	}
+
+	rows, err := db.Query(query)
+	if err != nil {
+		return nil
+	}
+	defer rows.Close()
+
+	var out []map[string]string
+	for rows.Next() {
+		cells := make([]sql.NullString, len(names))
+		ptrs := make([]any, len(names))
+		for i := range cells {
+			ptrs[i] = &cells[i]
+		}
+		if err := rows.Scan(ptrs...); err != nil {
+			return out
+		}
+		row := make(map[string]string, len(names))
+		for i, n := range names {
+			if cells[i].Valid {
+				row[n] = strings.TrimSpace(cells[i].String)
+			} else {
+				row[n] = ""
+			}
+		}
+		out = append(out, row)
+	}
+	return out
+}
+
+// quoteIdent quotes an SQLite identifier.
+func quoteIdent(s string) string {
+	return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
+}

internal/parser/parser.go

@@ -0,0 +1,194 @@
+// Package parser extracts attribute-table columns (with a few sample values)
+// from vector geo files: zipped ESRI shapefiles (.dbf), GeoJSON, and GeoPackage
+// (.gpkg). It is used to let a user pick the KATO column and map year columns.
+package parser
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"gis/internal/domain"
+)
+
+// sampleRows is the maximum number of sample values collected per column.
+const sampleRows = 5
+
+// ErrNoColumns indicates the file had no detectable attribute columns.
+var ErrNoColumns = errors.New("no attribute columns detected")
+
+// Columns detects the attribute columns of a vector file, dispatching on the
+// filename extension.
+func Columns(filename string, data []byte) ([]domain.AttributeColumn, error) {
+	switch ext := strings.ToLower(filepath.Ext(filename)); ext {
+	case ".zip":
+		return shapefileColumns(data)
+	case ".geojson", ".json":
+		return geojsonColumns(data)
+	case ".gpkg":
+		return gpkgColumns(data)
+	default:
+		return nil, fmt.Errorf("unsupported format %q", ext)
+	}
+}
+
+// Rows reads every feature's attribute values as name->value maps, dispatching
+// on the filename extension. Used to unpivot the attribute table.
+func Rows(filename string, data []byte) ([]map[string]string, error) {
+	switch ext := strings.ToLower(filepath.Ext(filename)); ext {
+	case ".zip":
+		return shapefileRows(data)
+	case ".geojson", ".json":
+		return geojsonRows(data)
+	case ".gpkg":
+		return gpkgRows(data)
+	default:
+		return nil, fmt.Errorf("unsupported format %q", ext)
+	}
+}
+
+func geojsonRows(data []byte) ([]map[string]string, error) {
+	var fc struct {
+		Features []struct {
+			Properties map[string]json.RawMessage `json:"properties"`
+		} `json:"features"`
+	}
+	if err := json.Unmarshal(data, &fc); err != nil {
+		return nil, fmt.Errorf("decode geojson: %w", err)
+	}
+
+	rows := make([]map[string]string, 0, len(fc.Features))
+	for _, f := range fc.Features {
+		row := make(map[string]string, len(f.Properties))
+		for k, raw := range f.Properties {
+			row[k] = rawToSample(raw)
+		}
+		rows = append(rows, row)
+	}
+	return rows, nil
+}
+
+func geojsonColumns(data []byte) ([]domain.AttributeColumn, error) {
+	var fc struct {
+		Features []struct {
+			Properties json.RawMessage `json:"properties"`
+		} `json:"features"`
+	}
+	if err := json.Unmarshal(data, &fc); err != nil {
+		return nil, fmt.Errorf("decode geojson: %w", err)
+	}
+	if len(fc.Features) == 0 {
+		return nil, ErrNoColumns
+	}
+
+	// Column order is taken from the first feature's properties.
+	keys, err := orderedKeys(fc.Features[0].Properties)
+	if err != nil {
+		return nil, err
+	}
+	if len(keys) == 0 {
+		return nil, ErrNoColumns
+	}
+
+	samples := make(map[string][]string, len(keys))
+	for i, f := range fc.Features {
+		if i >= sampleRows {
+			break
+		}
+		var props map[string]json.RawMessage
+		if err := json.Unmarshal(f.Properties, &props); err != nil {
+			continue
+		}
+		for _, k := range keys {
+			if raw, ok := props[k]; ok {
+				samples[k] = append(samples[k], rawToSample(raw))
+			}
+		}
+	}
+
+	cols := make([]domain.AttributeColumn, 0, len(keys))
+	for _, k := range keys {
+		cols = append(cols, domain.AttributeColumn{Name: k, Samples: samples[k]})
+	}
+	return cols, nil
+}
+
+// orderedKeys returns the keys of a JSON object in document order.
+func orderedKeys(obj json.RawMessage) ([]string, error) {
+	dec := json.NewDecoder(bytes.NewReader(obj))
+	t, err := dec.Token()
+	if err != nil {
+		return nil, err
+	}
+	if d, ok := t.(json.Delim); !ok || d != '{' {
+		return nil, fmt.Errorf("properties is not an object")
+	}
+
+	var keys []string
+	for dec.More() {
+		kt, err := dec.Token()
+		if err != nil {
+			return nil, err
+		}
+		key, ok := kt.(string)
+		if !ok {
+			return nil, fmt.Errorf("unexpected object key")
+		}
+		keys = append(keys, key)
+		if err := skipValue(dec); err != nil {
+			return nil, err
+		}
+	}
+	return keys, nil
+}
+
+// skipValue consumes the next JSON value (scalar, object, or array).
+func skipValue(dec *json.Decoder) error {
+	t, err := dec.Token()
+	if err != nil {
+		return err
+	}
+	d, ok := t.(json.Delim)
+	if !ok || (d != '{' && d != '[') {
+		return nil
+	}
+	depth := 1
+	for depth > 0 {
+		t, err := dec.Token()
+		if err != nil {
+			return err
+		}
+		if d, ok := t.(json.Delim); ok {
+			if d == '{' || d == '[' {
+				depth++
+			} else {
+				depth--
+			}
+		}
+	}
+	return nil
+}
+
+// rawToSample renders a JSON value as a short sample string.
+func rawToSample(raw json.RawMessage) string {
+	var v any
+	if err := json.Unmarshal(raw, &v); err != nil {
+		return strings.TrimSpace(string(raw))
+	}
+	switch t := v.(type) {
+	case nil:
+		return ""
+	case string:
+		return t
+	case float64:
+		return strconv.FormatFloat(t, 'f', -1, 64)
+	case bool:
+		return strconv.FormatBool(t)
+	default:
+		return strings.TrimSpace(string(raw))
+	}
+}

internal/parser/parser_test.go

@@ -0,0 +1,48 @@
+package parser
+
+import "testing"
+
+func TestColumns_GeoJSON(t *testing.T) {
+	data := []byte(`{
+		"type": "FeatureCollection",
+		"features": [
+			{"type":"Feature","properties":{"ObjectID":1,"F_2023":100,"D_2025":200,"като":"751010000"},"geometry":null},
+			{"type":"Feature","properties":{"ObjectID":2,"F_2023":150,"D_2025":250,"като":"751020000"},"geometry":null}
+		]
+	}`)
+
+	cols, err := Columns("regions.geojson", data)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	// Order must follow the first feature's properties.
+	wantNames := []string{"ObjectID", "F_2023", "D_2025", "като"}
+	if len(cols) != len(wantNames) {
+		t.Fatalf("want %d columns, got %d (%+v)", len(wantNames), len(cols), cols)
+	}
+	for i, want := range wantNames {
+		if cols[i].Name != want {
+			t.Errorf("column %d = %q, want %q", i, cols[i].Name, want)
+		}
+	}
+
+	// KATO column should carry sample values from both features.
+	kato := cols[3]
+	if len(kato.Samples) != 2 || kato.Samples[0] != "751010000" {
+		t.Errorf("unexpected kato samples: %v", kato.Samples)
+	}
+}
+
+func TestColumns_UnsupportedFormat(t *testing.T) {
+	if _, err := Columns("data.txt", []byte("x")); err == nil {
+		t.Fatal("expected error for unsupported format")
+	}
+}
+
+func TestColumns_GeoJSONNoFeatures(t *testing.T) {
+	_, err := Columns("empty.geojson", []byte(`{"type":"FeatureCollection","features":[]}`))
+	if err == nil {
+		t.Fatal("expected error for empty feature collection")
+	}
+}

internal/parser/shapefile.go

@@ -0,0 +1,169 @@
+package parser
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"unicode/utf8"
+
+	"gis/internal/domain"
+
+	"golang.org/x/text/encoding/charmap"
+)
+
+// readDBF extracts the .dbf bytes from a zipped ESRI shapefile.
+func readDBF(data []byte) ([]byte, error) {
+	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		return nil, fmt.Errorf("open zip: %w", err)
+	}
+
+	var dbf *zip.File
+	for _, f := range zr.File {
+		if strings.HasSuffix(strings.ToLower(f.Name), ".dbf") {
+			dbf = f
+			break
+		}
+	}
+	if dbf == nil {
+		return nil, errors.New("no .dbf file found in archive")
+	}
+
+	rc, err := dbf.Open()
+	if err != nil {
+		return nil, fmt.Errorf("open .dbf: %w", err)
+	}
+	defer rc.Close()
+
+	raw, err := io.ReadAll(rc)
+	if err != nil {
+		return nil, fmt.Errorf("read .dbf: %w", err)
+	}
+	return raw, nil
+}
+
+// shapefileColumns reads the .dbf attribute columns (with samples).
+func shapefileColumns(data []byte) ([]domain.AttributeColumn, error) {
+	raw, err := readDBF(data)
+	if err != nil {
+		return nil, err
+	}
+
+	fields, headerSize, recordLen, err := dbfHeader(raw)
+	if err != nil {
+		return nil, err
+	}
+
+	samples := make([][]string, len(fields))
+	collected := 0
+	dbfEachRecord(raw, fields, headerSize, recordLen, func(values []string) bool {
+		for i := range fields {
+			samples[i] = append(samples[i], values[i])
+		}
+		collected++
+		return collected < sampleRows
+	})
+
+	cols := make([]domain.AttributeColumn, len(fields))
+	for i, f := range fields {
+		cols[i] = domain.AttributeColumn{Name: f.name, Samples: samples[i]}
+	}
+	return cols, nil
+}
+
+// shapefileRows reads every record of the .dbf as a name->value map.
+func shapefileRows(data []byte) ([]map[string]string, error) {
+	raw, err := readDBF(data)
+	if err != nil {
+		return nil, err
+	}
+
+	fields, headerSize, recordLen, err := dbfHeader(raw)
+	if err != nil {
+		return nil, err
+	}
+
+	var rows []map[string]string
+	dbfEachRecord(raw, fields, headerSize, recordLen, func(values []string) bool {
+		row := make(map[string]string, len(fields))
+		for i, f := range fields {
+			row[f.name] = values[i]
+		}
+		rows = append(rows, row)
+		return true
+	})
+	return rows, nil
+}
+
+type dbfField struct {
+	name   string
+	offset int
+	length int
+}
+
+// dbfHeader parses a dBASE III/IV header into fields plus record geometry.
+func dbfHeader(b []byte) (fields []dbfField, headerSize, recordLen int, err error) {
+	if len(b) < 32 {
+		return nil, 0, 0, errors.New("dbf too short")
+	}
+	headerSize = int(binary.LittleEndian.Uint16(b[8:10]))
+	recordLen = int(binary.LittleEndian.Uint16(b[10:12]))
+
+	recOffset := 1 // first byte of each record is the deletion flag
+	for off := 32; off+32 <= len(b) && b[off] != 0x0D; off += 32 {
+		name := decodeText(trimNull(b[off : off+11]))
+		length := int(b[off+16])
+		fields = append(fields, dbfField{name: name, offset: recOffset, length: length})
+		recOffset += length
+	}
+	if len(fields) == 0 {
+		return nil, 0, 0, ErrNoColumns
+	}
+	return fields, headerSize, recordLen, nil
+}
+
+// dbfEachRecord decodes each non-deleted record's field values (in field order)
+// and calls fn; iteration stops when fn returns false.
+func dbfEachRecord(b []byte, fields []dbfField, headerSize, recordLen int, fn func(values []string) bool) {
+	if headerSize <= 0 || recordLen <= 0 {
+		return
+	}
+	for start := headerSize; start+recordLen <= len(b); start += recordLen {
+		rec := b[start : start+recordLen]
+		if rec[0] == '*' { // deleted record
+			continue
+		}
+		values := make([]string, len(fields))
+		for i, f := range fields {
+			if f.offset+f.length <= len(rec) {
+				values[i] = strings.TrimSpace(decodeText(rec[f.offset : f.offset+f.length]))
+			}
+		}
+		if !fn(values) {
+			return
+		}
+	}
+}
+
+func trimNull(b []byte) []byte {
+	if i := bytes.IndexByte(b, 0); i >= 0 {
+		return b[:i]
+	}
+	return b
+}
+
+// decodeText returns UTF-8 text, falling back to Windows-1251 (common for
+// Cyrillic KATO data) when the bytes are not valid UTF-8.
+func decodeText(b []byte) string {
+	if utf8.Valid(b) {
+		return string(b)
+	}
+	if decoded, err := charmap.Windows1251.NewDecoder().Bytes(b); err == nil {
+		return string(decoded)
+	}
+	return string(b)
+}