refactor!: make addrkit a multi-city address normalizer

- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
2026-10-27 14:35:51 +07:00
parent a0b4cb2b14
commit 17ff5c591f
38 changed files with 2978 additions and 1164 deletions
@@ -29,7 +29,7 @@ func TestRealRegistryParseRate(t *testing.T) {
 	if err != nil {
 		t.Fatalf("read registry: %v", err)
 	}
-	entries, stats := Build(sources)
+	entries, stats := Build(sources, BuildOptions{})

 	rate := 0.0
 	if stats.Total > 0 {
@@ -20,24 +20,38 @@ type Source struct {
 	ID       string
 	Address  string
 	District string
+	// City is optional: when the dump is already split per city it may be
+	// omitted and supplied as the default city to Build.
+	City string
 }

 // Stats summarises a build and, crucially, surfaces rows the parser could not
 // structure. A non-empty UnparsedCount is a data-quality signal worth alerting
 // on, not an error.
 type Stats struct {
-	Total           int      `json:"total"`
-	Parsed          int      `json:"parsed"`
-	Unparsed        int      `json:"unparsed"`
-	Duplicates      int      `json:"duplicates"`
-	DistinctStreets int      `json:"distinct_streets"`
-	UnparsedSample  []string `json:"unparsed_sample,omitempty"`
+	Total           int            `json:"total"`
+	Parsed          int            `json:"parsed"`
+	Unparsed        int            `json:"unparsed"`
+	Duplicates      int            `json:"duplicates"`
+	DistinctStreets int            `json:"distinct_streets"`
+	Cities          map[string]int `json:"cities,omitempty"`
+	UnparsedSample  []string       `json:"unparsed_sample,omitempty"`
+}
+
+// BuildOptions controls how source rows become canonical entries.
+type BuildOptions struct {
+	// Parser defaults to address.DefaultParser().
+	Parser *address.Parser
+	// DefaultCity is used for rows that neither carry a city field nor name a
+	// city in the address. Leave it empty for multi-city dumps where every row
+	// is expected to identify its city.
+	DefaultCity string
 }

 // ReadBulkNDJSON reads an Elasticsearch Bulk NDJSON dump: pairs of an action
 // line ({"index":{"_id":...}}) followed by a source line
-// ({"address":...,"district_name":...}). Bare source lines (no action line)
-// are also accepted, which makes plain JSONL usable too.
+// ({"address":...,"district_name":...,"city":...}). Bare source lines (no
+// action line) are also accepted, which makes plain JSONL usable too.
 func ReadBulkNDJSON(r io.Reader) ([]Source, error) {
 	sc := bufio.NewScanner(r)
 	// 1 MiB per line is far beyond any realistic address row.
@@ -75,6 +89,7 @@ func ReadBulkNDJSON(r io.Reader) ([]Source, error) {
 			if d, ok := obj["district_name"]; ok {
 				_ = json.Unmarshal(d, &src.District)
 			}
+			src.City = firstString(obj, "city", "city_name")
 			sources = append(sources, src)
 			pendingID = ""
 		}
@@ -85,17 +100,40 @@ func ReadBulkNDJSON(r io.Reader) ([]Source, error) {
 	return sources, nil
 }

+// firstString returns the first non-empty string field among names.
+func firstString(obj map[string]json.RawMessage, names ...string) string {
+	for _, name := range names {
+		raw, ok := obj[name]
+		if !ok {
+			continue
+		}
+		var s string
+		if err := json.Unmarshal(raw, &s); err == nil && strings.TrimSpace(s) != "" {
+			return s
+		}
+	}
+	return ""
+}
+
 // Build parses every source row into a canonical Entry and reports statistics.
 // Rows without a house number are still indexed for street lookups but counted
 // as unparsed, since a registry row without a house is suspicious.
-func Build(sources []Source) ([]address.Entry, Stats) {
+func Build(sources []Source, opts BuildOptions) ([]address.Entry, Stats) {
+	parser := opts.Parser
+	if parser == nil {
+		parser = address.NewParser(address.DefaultDictionary(), opts.DefaultCity)
+	}
 	entries := make([]address.Entry, 0, len(sources))
 	streets := make(map[string]struct{})
 	seen := make(map[string]struct{})
-	stats := Stats{Total: len(sources)}
+	stats := Stats{Total: len(sources), Cities: make(map[string]int)}

 	for _, src := range sources {
-		p := address.Parse(src.Address)
+		// Precedence: an explicit source field wins over a city found in the
+		// address text; the parser's default city is the last resort. Passing
+		// an empty hint is what lets the parser apply that precedence.
+		cityHint := strings.TrimSpace(src.City)
+		p := parser.ParseInCity(src.Address, cityHint)
 		if p.NameKey == "" {
 			stats.Unparsed++
 			appendSample(&stats, src.Address)
@@ -110,9 +148,11 @@ func Build(sources []Source) ([]address.Entry, Stats) {
 		}

 		entry := address.Entry{
-			ID:             src.ID,
+			ID:             namespacedID(src.ID, p.CityKey),
 			Address:        src.Address,
 			District:       src.District,
+			City:           p.City,
+			CityKey:        p.CityKey,
 			Ordinal:        p.Ordinal,
 			NameKey:        p.NameKey,
 			NameDisplay:    p.NameDisplay,
@@ -128,9 +168,8 @@ func Build(sources []Source) ([]address.Entry, Stats) {
 		}

 		// The source registry contains rows that differ only by "стр."
-		// (structure). Since structure does not affect which house an outage
-		// belongs to, drop such duplicates rather than surfacing them as
-		// spurious ambiguity at query time.
+		// (structure). They are distinct buildings, so structure is part of
+		// the de-duplication key and an exact structure query can be honoured.
 		if key := dedupKey(entry); key != "" {
 			if _, ok := seen[key]; ok {
 				stats.Duplicates++
@@ -141,24 +180,41 @@ func Build(sources []Source) ([]address.Entry, Stats) {

 		entries = append(entries, entry)
 		streets[p.StreetKey] = struct{}{}
+		stats.Cities[cityLabel(p.City)]++
 	}
 	stats.DistinctStreets = len(streets)
 	sort.Strings(stats.UnparsedSample)
 	return entries, stats
 }

-// dedupKey identifies the same house unit within the same street, ignoring the
-// structure component.
+// namespacedID keeps source ids unique when several city registries are merged
+// into one database. Source systems often number houses independently.
+func namespacedID(id, cityKey string) string {
+	if id == "" || cityKey == "" {
+		return id
+	}
+	return cityKey + "/" + id
+}
+
+// dedupKey identifies the same house unit within the same city and street,
+// including its structure and building components.
 func dedupKey(e address.Entry) string {
 	if e.House == "" {
 		return ""
 	}
 	return strings.Join([]string{
-		e.StreetKey, e.House, e.HouseFrac, e.HouseLetter,
-		e.Building, e.BuildingFrac, e.BuildingLetter,
+		e.CityKey, e.StreetKey, e.House, e.HouseFrac, e.HouseLetter,
+		e.Building, e.BuildingFrac, e.BuildingLetter, e.Structure,
 	}, "|")
 }

+func cityLabel(city string) string {
+	if city == "" {
+		return "(none)"
+	}
+	return city
+}
+
 func appendSample(s *Stats, address string) {
 	const maxSamples = 20
 	if len(s.UnparsedSample) < maxSamples {
@@ -3,11 +3,13 @@ package seed
 import (
 	"strings"
 	"testing"
+
+	"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
 )

 func TestReadBulkNDJSON(t *testing.T) {
 	input := `{"index": {"_index": "address_registry", "_id": "a1"}}
-{"address": "Станционная улица 1/1", "district_name": "Ленинский"}
+{"address": "Станционная улица 1/1", "district_name": "Ленинский", "city": "Новосибирск"}
 {"index": {"_id": "a2"}}
 {"address": "Искитимская улица 5", "district_name": "Кировский"}

@@ -20,7 +22,7 @@ func TestReadBulkNDJSON(t *testing.T) {
 	if len(src) != 3 {
 		t.Fatalf("got %d sources, want 3", len(src))
 	}
-	if src[0].ID != "a1" || src[0].District != "Ленинский" {
+	if src[0].ID != "a1" || src[0].District != "Ленинский" || src[0].City != "Новосибирск" {
 		t.Errorf("first source = %+v", src[0])
 	}
 	if src[2].Address != "Третья улица 7" {
@@ -34,7 +36,7 @@ func TestBuild(t *testing.T) {
 		{ID: "2", Address: "Искитимская улица 5"},
 		{ID: "3", Address: "Станционная улица"}, // no house -> unparsed
 	}
-	entries, stats := Build(sources)
+	entries, stats := Build(sources, BuildOptions{})

 	if stats.Total != 3 {
 		t.Errorf("Total = %d, want 3", stats.Total)
@@ -53,3 +55,71 @@ func TestBuild(t *testing.T) {
 		t.Errorf("DistinctStreets = %d, want 2", stats.DistinctStreets)
 	}
 }
+
+func TestBuildKeepsCitiesApart(t *testing.T) {
+	sources := []Source{
+		{ID: "1", Address: "Ленина улица 5", City: "Новосибирск"},
+		{ID: "1", Address: "Ленина улица 5", City: "Бердск"},
+		{ID: "2", Address: "Ленина улица 7", City: "Новосибирск"},
+	}
+	entries, stats := Build(sources, BuildOptions{})
+
+	if len(entries) != 3 {
+		t.Fatalf("entries = %d, want 3: %+v", len(entries), entries)
+	}
+	if stats.DistinctStreets != 2 {
+		t.Errorf("DistinctStreets = %d, want 2 (same street in two cities)", stats.DistinctStreets)
+	}
+	if entries[0].StreetKey == entries[1].StreetKey {
+		t.Errorf("streets from different cities share a key: %q", entries[0].StreetKey)
+	}
+	if entries[0].ID == entries[1].ID {
+		t.Errorf("source ids collided across cities: %q", entries[0].ID)
+	}
+	if stats.Cities["Новосибирск"] != 2 || stats.Cities["Бердск"] != 1 {
+		t.Errorf("city stats = %v", stats.Cities)
+	}
+}
+
+func TestBuildAppliesDefaultCity(t *testing.T) {
+	sources := []Source{{ID: "1", Address: "Станционная улица 1/1"}}
+	entries, _ := Build(sources, BuildOptions{DefaultCity: "Новосибирск"})
+	if len(entries) != 1 {
+		t.Fatalf("entries = %d", len(entries))
+	}
+	if entries[0].City != "Новосибирск" || entries[0].CityKey != "новосибирск" {
+		t.Errorf("city = %q / %q", entries[0].City, entries[0].CityKey)
+	}
+}
+
+func TestBuildKeepsStructuresDistinct(t *testing.T) {
+	sources := []Source{
+		{ID: "1", Address: "Станционная улица 1"},
+		{ID: "2", Address: "Станционная улица 1 стр. 2"},
+	}
+	entries, stats := Build(sources, BuildOptions{})
+
+	if len(entries) != 2 {
+		t.Fatalf("entries = %d, want 2 (structure is part of identity)", len(entries))
+	}
+	if stats.Duplicates != 0 {
+		t.Errorf("Duplicates = %d, want 0", stats.Duplicates)
+	}
+	if entries[0].Structure == entries[1].Structure {
+		t.Errorf("structures collapsed: %+v", entries)
+	}
+}
+
+func TestBuildParsesCityFromAddress(t *testing.T) {
+	parser := address.NewParser(address.NewDictionary([]address.City{
+		{Name: "Новосибирск", Aliases: []string{"нск", "новосибирская область"}},
+	}), "")
+	sources := []Source{{ID: "1", Address: "нск, Станционная улица 1/1"}}
+	entries, _ := Build(sources, BuildOptions{Parser: parser})
+	if len(entries) != 1 {
+		t.Fatalf("entries = %d", len(entries))
+	}
+	if entries[0].City != "Новосибирск" || entries[0].NameKey != "станционная" {
+		t.Errorf("entry = %+v", entries[0])
+	}
+}