addrkit: deterministic address resolution service (Go)

- domain: address normalization/parsing, matching with explicit statuses
- adapters: SQLite registry store, ArcGIS outages client, optional LLM rewriter, HTTP API
- seed pipeline with duplicate handling and unparsed reporting
- CLI (seed/verify/resolve), systemd unit, OpenAPI contract, tests

internal/seed/seed.go

@@ -0,0 +1,165 @@
+// Package seed turns a registry dump into canonical address entries. It is the
+// only place that knows how the source data is shaped.
+//
+// The same parser used at query time is used here, so the index and the query
+// never disagree about how an address decomposes.
+package seed
+
+import (
+	"bufio"
+	"encoding/json"
+	"io"
+	"sort"
+	"strings"
+
+	"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
+)
+
+// Source is one raw registry row.
+type Source struct {
+	ID       string
+	Address  string
+	District string
+}
+
+// Stats summarises a build and, crucially, surfaces rows the parser could not
+// structure. A non-empty UnparsedCount is a data-quality signal worth alerting
+// on, not an error.
+type Stats struct {
+	Total           int      `json:"total"`
+	Parsed          int      `json:"parsed"`
+	Unparsed        int      `json:"unparsed"`
+	Duplicates      int      `json:"duplicates"`
+	DistinctStreets int      `json:"distinct_streets"`
+	UnparsedSample  []string `json:"unparsed_sample,omitempty"`
+}
+
+// ReadBulkNDJSON reads an Elasticsearch Bulk NDJSON dump: pairs of an action
+// line ({"index":{"_id":...}}) followed by a source line
+// ({"address":...,"district_name":...}). Bare source lines (no action line)
+// are also accepted, which makes plain JSONL usable too.
+func ReadBulkNDJSON(r io.Reader) ([]Source, error) {
+	sc := bufio.NewScanner(r)
+	// 1 MiB per line is far beyond any realistic address row.
+	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
+
+	var (
+		sources   []Source
+		pendingID string
+	)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" {
+			continue
+		}
+		var obj map[string]json.RawMessage
+		if err := json.Unmarshal([]byte(line), &obj); err != nil {
+			continue // skip malformed lines rather than aborting the whole seed
+		}
+
+		if action, ok := obj["index"]; ok {
+			var a struct {
+				ID string `json:"_id"`
+			}
+			_ = json.Unmarshal(action, &a)
+			pendingID = a.ID
+			continue
+		}
+
+		if addrRaw, ok := obj["address"]; ok {
+			var addr string
+			if err := json.Unmarshal(addrRaw, &addr); err != nil || addr == "" {
+				continue
+			}
+			src := Source{ID: pendingID, Address: addr}
+			if d, ok := obj["district_name"]; ok {
+				_ = json.Unmarshal(d, &src.District)
+			}
+			sources = append(sources, src)
+			pendingID = ""
+		}
+	}
+	if err := sc.Err(); err != nil {
+		return nil, err
+	}
+	return sources, nil
+}
+
+// Build parses every source row into a canonical Entry and reports statistics.
+// Rows without a house number are still indexed for street lookups but counted
+// as unparsed, since a registry row without a house is suspicious.
+func Build(sources []Source) ([]address.Entry, Stats) {
+	entries := make([]address.Entry, 0, len(sources))
+	streets := make(map[string]struct{})
+	seen := make(map[string]struct{})
+	stats := Stats{Total: len(sources)}
+
+	for _, src := range sources {
+		p := address.Parse(src.Address)
+		if p.NameKey == "" {
+			stats.Unparsed++
+			appendSample(&stats, src.Address)
+			continue
+		}
+		entry := address.Entry{
+			ID:             src.ID,
+			Address:        src.Address,
+			District:       src.District,
+			Ordinal:        p.Ordinal,
+			NameKey:        p.NameKey,
+			NameDisplay:    p.NameDisplay,
+			StreetType:     p.StreetType,
+			StreetKey:      p.StreetKey,
+			House:          p.House,
+			HouseFrac:      p.HouseFrac,
+			HouseLetter:    p.HouseLetter,
+			Building:       p.Building,
+			BuildingFrac:   p.BuildingFrac,
+			BuildingLetter: p.BuildingLetter,
+			Structure:      p.Structure,
+		}
+
+		// The source registry contains rows that differ only by "стр."
+		// (structure). Since structure does not affect which house an outage
+		// belongs to, drop such duplicates rather than surfacing them as
+		// spurious ambiguity at query time.
+		if key := dedupKey(entry); key != "" {
+			if _, ok := seen[key]; ok {
+				stats.Duplicates++
+				continue
+			}
+			seen[key] = struct{}{}
+		}
+
+		entries = append(entries, entry)
+		streets[p.StreetKey] = struct{}{}
+		if p.HasHouse {
+			stats.Parsed++
+		} else {
+			stats.Unparsed++
+			appendSample(&stats, src.Address)
+		}
+	}
+	stats.DistinctStreets = len(streets)
+	sort.Strings(stats.UnparsedSample)
+	return entries, stats
+}
+
+// dedupKey identifies the same house unit within the same street, ignoring the
+// structure component.
+func dedupKey(e address.Entry) string {
+	if e.House == "" {
+		return ""
+	}
+	return strings.Join([]string{
+		e.StreetKey, e.House, e.HouseFrac, e.HouseLetter,
+		e.Building, e.BuildingFrac, e.BuildingLetter,
+	}, "|")
+}
+
+func appendSample(s *Stats, address string) {
+	const maxSamples = 20
+	if len(s.UnparsedSample) < maxSamples {
+		s.UnparsedSample = append(s.UnparsedSample, address)
+	}
+}