refactor!: make addrkit a multi-city address normalizer

- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
2026-10-27 14:35:51 +07:00
parent a0b4cb2b14
commit 17ff5c591f
38 changed files with 2978 additions and 1164 deletions
@@ -9,6 +9,7 @@ package match

 import (
 	"sort"
+	"unicode/utf8"

 	"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
 )
@@ -16,16 +17,19 @@ import (
 // Candidate is one plausible match, returned alongside a non-"found" result so
 // the caller (or an operator) can see what the resolver was choosing between.
 type Candidate struct {
-	Address  string  `json:"address"`
-	District string  `json:"district,omitempty"`
-	House    string  `json:"house,omitempty"`
-	Score    float64 `json:"score"`
+	Address   string  `json:"address"`
+	City      string  `json:"city,omitempty"`
+	District  string  `json:"district,omitempty"`
+	House     string  `json:"house,omitempty"`
+	Structure string  `json:"structure,omitempty"`
+	Score     float64 `json:"score"`
 }

 // Result is the outcome of a resolution.
 type Result struct {
 	Status      address.Status `json:"status"`
 	Address     string         `json:"address,omitempty"`
+	City        string         `json:"city,omitempty"`
 	District    string         `json:"district,omitempty"`
 	Confidence  float64        `json:"confidence"`
 	Candidates  []Candidate    `json:"candidates,omitempty"`
@@ -59,10 +63,10 @@ func DefaultOptions() Options {
 }

 type streetRef struct {
-	key        string
-	ordinal    string
-	nameKey    string
-	streetType string
+	key     string
+	cityKey string
+	ordinal string
+	nameKey string
 }

 // Index is an immutable, in-memory registry index. It is safe for concurrent
@@ -72,26 +76,46 @@ type Index struct {
 	opts    Options
 	byKey   map[string][]address.Entry
 	streets []streetRef
+	cities  map[string]string // cityKey -> canonical display name
+	size    int
 }

+// maxFuzzyNameRunes bounds the fuzzy scan. No real street name is longer than
+// this; without the bound a single huge token would make every Levenshtein
+// comparison walk a very long string and turn a request into a CPU sink.
+const maxFuzzyNameRunes = 96
+
 // NewIndex builds an index from canonical entries. Entries sharing a StreetKey
 // are grouped; distinct streets are collected for typo-tolerant matching.
+//
+// Entries are sorted first, so candidate order is deterministic regardless of
+// the order in which the storage adapter returned them.
 func NewIndex(version string, entries []address.Entry, opts Options) *Index {
+	sorted := append([]address.Entry(nil), entries...)
+	sort.SliceStable(sorted, func(i, j int) bool { return entryLess(sorted[i], sorted[j]) })
+
 	ix := &Index{
 		version: version,
 		opts:    opts,
-		byKey:   make(map[string][]address.Entry, len(entries)),
+		byKey:   make(map[string][]address.Entry, len(sorted)),
+		cities:  make(map[string]string),
+		size:    len(sorted),
 	}
-	seen := make(map[string]struct{}, len(entries))
-	for _, e := range entries {
+	seen := make(map[string]struct{}, len(sorted))
+	for _, e := range sorted {
 		ix.byKey[e.StreetKey] = append(ix.byKey[e.StreetKey], e)
+		if e.CityKey != "" {
+			if _, ok := ix.cities[e.CityKey]; !ok {
+				ix.cities[e.CityKey] = e.City
+			}
+		}
 		if _, ok := seen[e.StreetKey]; !ok {
 			seen[e.StreetKey] = struct{}{}
 			ix.streets = append(ix.streets, streetRef{
-				key:        e.StreetKey,
-				ordinal:    e.Ordinal,
-				nameKey:    e.NameKey,
-				streetType: e.StreetType,
+				key:     e.StreetKey,
+				cityKey: e.CityKey,
+				ordinal: e.Ordinal,
+				nameKey: e.NameKey,
 			})
 		}
 	}
@@ -99,22 +123,57 @@ func NewIndex(version string, entries []address.Entry, opts Options) *Index {
 	return ix
 }

+// entryLess defines the canonical order of registry rows. Address strings are
+// the last meaningful tie-breaker; ID makes the order total even for duplicated
+// rows.
+func entryLess(a, b address.Entry) bool {
+	switch {
+	case a.CityKey != b.CityKey:
+		return a.CityKey < b.CityKey
+	case a.StreetKey != b.StreetKey:
+		return a.StreetKey < b.StreetKey
+	case a.House != b.House:
+		return a.House < b.House
+	case a.HouseFrac != b.HouseFrac:
+		return a.HouseFrac < b.HouseFrac
+	case a.HouseLetter != b.HouseLetter:
+		return a.HouseLetter < b.HouseLetter
+	case a.Building != b.Building:
+		return a.Building < b.Building
+	case a.BuildingFrac != b.BuildingFrac:
+		return a.BuildingFrac < b.BuildingFrac
+	case a.BuildingLetter != b.BuildingLetter:
+		return a.BuildingLetter < b.BuildingLetter
+	case a.Structure != b.Structure:
+		return a.Structure < b.Structure
+	case a.Address != b.Address:
+		return a.Address < b.Address
+	default:
+		return a.ID < b.ID
+	}
+}
+
 // Version returns the data version the index was built from.
 func (ix *Index) Version() string { return ix.version }

 // Size returns the number of indexed entries.
-func (ix *Index) Size() int {
-	n := 0
-	for _, v := range ix.byKey {
-		n += len(v)
+func (ix *Index) Size() int { return ix.size }
+
+// Cities returns the distinct canonical city names present in the index, in
+// deterministic order. A city-less index returns an empty slice.
+func (ix *Index) Cities() []string {
+	out := make([]string, 0, len(ix.cities))
+	for _, name := range ix.cities {
+		out = append(out, name)
 	}
-	return n
+	sort.Strings(out)
+	return out
 }

 // Resolve resolves a parsed address. It never returns an address that is not
 // present in the index.
 func (ix *Index) Resolve(p address.ParsedAddress) Result {
-	res := Result{DataVersion: ix.version}
+	res := Result{DataVersion: ix.version, City: p.City}

 	entries := ix.byKey[p.StreetKey]
 	similarity := 1.0
@@ -133,6 +192,11 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
 		res.Status = address.StatusNotFound
 		return res
 	}
+	if res.City == "" {
+		// A city-less query may still match exactly one city; report it so the
+		// caller does not have to re-derive the city from the candidates.
+		res.City = commonCity(entries)
+	}

 	if !p.HasHouse {
 		// Street is known but the caller did not supply a house: safe but
@@ -148,6 +212,7 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
 	case 1:
 		res.Status = address.StatusFound
 		res.Address = matched[0].Address
+		res.City = matched[0].City
 		res.District = matched[0].District
 		res.Confidence = clamp01(similarity)
 		return res
@@ -162,7 +227,14 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
 		}
 		res.Status = address.StatusPartial
 		res.Confidence = clamp01(similarity * 0.9)
-		res.Candidates = candidates(entries, similarity, ix.opts.MaxCandidates)
+		// Prefer houses that share the number/fraction/letter: they are the
+		// plausible corrections (a different corpus or structure), while the
+		// rest of the street is noise.
+		related := matchHouseUnitAll(entries, p)
+		if len(related) == 0 {
+			related = entries
+		}
+		res.Candidates = candidates(related, similarity, ix.opts.MaxCandidates)
 		return res
 	default:
 		res.Status = address.StatusAmbiguous
@@ -176,12 +248,16 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
 // the best street when the match is unambiguous, or the list of competing
 // street keys when several are within the ambiguity margin.
 func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry, similarity float64, ambiguousKeys []string) {
-	if p.NameKey == "" {
+	if p.NameKey == "" || utf8.RuneCountInString(p.NameKey) > maxFuzzyNameRunes {
 		return nil, 0, nil
 	}
 	best := 0.0
 	bestKey := ""
 	for _, s := range ix.streets {
+		// A query that names a city may only match streets in that city.
+		if p.CityKey != "" && s.cityKey != p.CityKey {
+			continue
+		}
 		// If the query carries an ordinal, the candidate's ordinal must match:
 		// a user who says "1-й Северный" does not mean the plain "Северная".
 		if !ordinalCompatible(p.Ordinal, s.ordinal) {
@@ -197,6 +273,9 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
 	}

 	for _, s := range ix.streets {
+		if p.CityKey != "" && s.cityKey != p.CityKey {
+			continue
+		}
 		if !ordinalCompatible(p.Ordinal, s.ordinal) {
 			continue
 		}
@@ -232,10 +311,12 @@ func (ix *Index) representatives(keys []string, score float64) []Candidate {
 		}
 		e := es[0]
 		out = append(out, Candidate{
-			Address:  e.Address,
-			District: e.District,
-			House:    e.House,
-			Score:    clamp01(score),
+			Address:   e.Address,
+			City:      e.City,
+			District:  e.District,
+			House:     e.House,
+			Structure: e.Structure,
+			Score:     clamp01(score),
 		})
 		if len(out) == ix.opts.MaxCandidates {
 			break
@@ -275,13 +356,16 @@ func sameHouseSeveral(entries []address.Entry, p address.ParsedAddress) bool {

 // houseUnitEqual compares the parts of a house unit. Unspecified (empty) parts
 // must also be empty on the entry side, so "60/1" never matches "60/1 к148".
+// The structure ("стр. 2") is part of the identity: a caller who explicitly
+// asked for a different structure must not be handed another building.
 func houseUnitEqual(e address.Entry, p address.ParsedAddress) bool {
 	return e.House == p.House &&
 		e.HouseFrac == p.HouseFrac &&
 		e.HouseLetter == p.HouseLetter &&
 		e.Building == p.Building &&
 		e.BuildingFrac == p.BuildingFrac &&
-		e.BuildingLetter == p.BuildingLetter
+		e.BuildingLetter == p.BuildingLetter &&
+		e.Structure == p.Structure
 }

 func candidates(entries []address.Entry, score float64, limit int) []Candidate {
@@ -291,10 +375,12 @@ func candidates(entries []address.Entry, score float64, limit int) []Candidate {
 	out := make([]Candidate, 0, len(entries))
 	for _, e := range entries {
 		out = append(out, Candidate{
-			Address:  e.Address,
-			District: e.District,
-			House:    e.House,
-			Score:    clamp01(score),
+			Address:   e.Address,
+			City:      e.City,
+			District:  e.District,
+			House:     e.House,
+			Structure: e.Structure,
+			Score:     clamp01(score),
 		})
 		if len(out) == limit {
 			break
@@ -312,3 +398,22 @@ func clamp01(v float64) float64 {
 	}
 	return v
 }
+
+// commonCity returns the city shared by all entries, or "" when they span
+// cities (or carry no city at all).
+func commonCity(entries []address.Entry) string {
+	key, name := "", ""
+	for _, e := range entries {
+		if e.CityKey == "" {
+			continue
+		}
+		if key == "" {
+			key, name = e.CityKey, e.City
+			continue
+		}
+		if e.CityKey != key {
+			return ""
+		}
+	}
+	return name
+}