refactor!: make addrkit a multi-city address normalizer

- remove the events/outages module, ArcGIS adapter, portal config and event statuses
- add a configurable city dictionary/parser and make City part of StreetKey
- seed: read city/city_name, namespace IDs by city, keep structures distinct
- fix reliability: schema v2 with fail-closed load, deterministic ordering,
  input limits, config validation, atomic DB publish, LLM response caps
- add read-only shadow mode comparing the old ES path with addrkit
- breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
Maksim Totmin
2026-10-27 14:35:51 +07:00
parent a0b4cb2b14
commit 17ff5c591f
38 changed files with 2978 additions and 1164 deletions
+36 -10
View File
@@ -26,8 +26,12 @@ import (
// information (ordinal, house letter, fraction) do not require an exact match;
// for them a different, real address is "lossy" rather than "wrong".
type Case struct {
Query string
Expect string
Query string
Expect string
// ExpectCity is the canonical city of the source entry. Empty for a
// city-less index; otherwise a "found" result in another city is a safety
// violation even when the street and house text look identical.
ExpectCity string
Category string
RequireExact bool
}
@@ -41,6 +45,7 @@ type CategoryStat struct {
Ambiguous int `json:"ambiguous"`
NotFound int `json:"not_found"`
WrongHouse int `json:"wrong_house"`
WrongCity int `json:"wrong_city"`
WrongStreet int `json:"wrong_street"`
Lossy int `json:"lossy"`
}
@@ -72,11 +77,19 @@ func Run(resolver *match.Resolver, cases []Case) Report {
case address.StatusFound:
rp := address.Parse(res.Address)
equivalent := rp.StreetKey == exp.StreetKey && sameUnit(rp, exp)
sameCity := c.ExpectCity == "" || res.City == c.ExpectCity
switch {
case equivalent:
case equivalent && sameCity:
st.Exact++
rep.Exact++
exact = true
case !sameCity:
// A wrong city is always a safety violation, even when the
// case is allowed to be lossy: the address text can be
// identical across cities.
st.WrongCity++
rep.Wrong++
rep.WrongExamples = appendExample(rep.WrongExamples, c, res.Address)
case !c.RequireExact:
// Missing information resolved to a different real address:
// expected, not a safety violation.
@@ -111,14 +124,17 @@ func Run(resolver *match.Resolver, cases []Case) Report {
return rep
}
// sameUnit compares house units, ignoring the structure component.
// sameUnit compares house units, including the structure: two buildings that
// differ only by "стр." are different addresses.
func sameUnit(a, b address.ParsedAddress) bool {
return a.House == b.House &&
return a.CityKey == b.CityKey &&
a.House == b.House &&
a.HouseFrac == b.HouseFrac &&
a.HouseLetter == b.HouseLetter &&
a.Building == b.Building &&
a.BuildingFrac == b.BuildingFrac &&
a.BuildingLetter == b.BuildingLetter
a.BuildingLetter == b.BuildingLetter &&
a.Structure == b.Structure
}
func appendExample(examples []string, c Case, got string) []string {
@@ -185,16 +201,24 @@ func perturbations(e address.Entry, rng *rand.Rand) []Case {
var out []Case
add := func(query, category string, requireExact bool) {
if query != "" && query != base {
out = append(out, Case{Query: query, Expect: base, Category: category, RequireExact: requireExact})
out = append(out, Case{
Query: query, Expect: base, ExpectCity: e.City,
Category: category, RequireExact: requireExact,
})
}
}
// Baseline: the canonical address itself must always resolve exactly.
out = append(out, Case{Query: base, Expect: base, Category: "baseline", RequireExact: true})
out = append(out, Case{
Query: base, Expect: base, ExpectCity: e.City,
Category: "baseline", RequireExact: true,
})
add(strings.ToLower(base), "case_lower", true)
add(strings.ToUpper(base), "case_upper", true)
add("г. Новосибирск, "+base, "city_prefix", true)
if e.City != "" {
add("г. "+e.City+", "+base, "city_prefix", true)
}
add(insertDom(base, e.House), "filler_dom", true)
add(corruptPunctuation(base), "punctuation", true)
add(addYo(base), "yo", true)
@@ -212,7 +236,9 @@ func perturbations(e address.Entry, rng *rand.Rand) []Case {
add(strings.Replace(base, "к"+e.Building, "корпус "+e.Building, 1), "corpus_synonym", true)
}
if hasStructure(base) {
add(dropStructure(base), "structure_drop", true)
// Dropping "стр. N" discards information: the base house may or may
// not exist on its own, so a different real address is acceptable.
add(dropStructure(base), "structure_drop", false)
}
if q := mutateLongestToken(base, rng); q != "" {
add(q, "typo", true)
+37
View File
@@ -4,6 +4,9 @@ import (
"math/rand"
"testing"
"unicode/utf8"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
)
// TestMoveTypeFront guards against the slice-aliasing regression that made the
@@ -47,3 +50,37 @@ func TestMutateOnlyNameTokens(t *testing.T) {
}
}
}
// TestRunDetectsWrongCity guards the multi-city safety net: canonical address
// text is identical across cities, so a found result must be checked against
// the expected city, not only against street/house text.
func TestRunDetectsWrongCity(t *testing.T) {
parser := address.NewParser(address.NewDictionary([]address.City{{Name: "Бердск"}}), "")
p := parser.ParseInCity("Ленина улица 5", "Бердск")
entries := []address.Entry{{
ID: "b1", Address: "Ленина улица 5",
City: p.City, CityKey: p.CityKey,
Ordinal: p.Ordinal, NameKey: p.NameKey, StreetKey: p.StreetKey, House: p.House,
}}
resolver := match.NewResolverWithOptions(
match.NewIndex("test", entries, match.DefaultOptions()),
nil,
match.ResolverOptions{Parser: parser},
)
cases := []Case{
{Query: "Ленина 5", Expect: "Ленина улица 5", ExpectCity: "Новосибирск", Category: "wrong_city", RequireExact: true},
{Query: "Ленина 5", Expect: "Ленина улица 5", ExpectCity: "Бердск", Category: "right_city", RequireExact: true},
}
rep := Run(resolver, cases)
if got := rep.Categories["wrong_city"]; got.WrongCity != 1 || got.Exact != 0 {
t.Fatalf("wrong_city = %+v, want WrongCity=1 Exact=0", got)
}
if got := rep.Categories["right_city"]; got.Exact != 1 || got.WrongCity != 0 {
t.Fatalf("right_city = %+v, want Exact=1 WrongCity=0", got)
}
if rep.Wrong != 1 {
t.Fatalf("report Wrong = %d, want 1", rep.Wrong)
}
}