refactor!: make addrkit a multi-city address normalizer
- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
+36
-10
@@ -26,8 +26,12 @@ import (
|
||||
// information (ordinal, house letter, fraction) do not require an exact match;
|
||||
// for them a different, real address is "lossy" rather than "wrong".
|
||||
type Case struct {
|
||||
Query string
|
||||
Expect string
|
||||
Query string
|
||||
Expect string
|
||||
// ExpectCity is the canonical city of the source entry. Empty for a
|
||||
// city-less index; otherwise a "found" result in another city is a safety
|
||||
// violation even when the street and house text look identical.
|
||||
ExpectCity string
|
||||
Category string
|
||||
RequireExact bool
|
||||
}
|
||||
@@ -41,6 +45,7 @@ type CategoryStat struct {
|
||||
Ambiguous int `json:"ambiguous"`
|
||||
NotFound int `json:"not_found"`
|
||||
WrongHouse int `json:"wrong_house"`
|
||||
WrongCity int `json:"wrong_city"`
|
||||
WrongStreet int `json:"wrong_street"`
|
||||
Lossy int `json:"lossy"`
|
||||
}
|
||||
@@ -72,11 +77,19 @@ func Run(resolver *match.Resolver, cases []Case) Report {
|
||||
case address.StatusFound:
|
||||
rp := address.Parse(res.Address)
|
||||
equivalent := rp.StreetKey == exp.StreetKey && sameUnit(rp, exp)
|
||||
sameCity := c.ExpectCity == "" || res.City == c.ExpectCity
|
||||
switch {
|
||||
case equivalent:
|
||||
case equivalent && sameCity:
|
||||
st.Exact++
|
||||
rep.Exact++
|
||||
exact = true
|
||||
case !sameCity:
|
||||
// A wrong city is always a safety violation, even when the
|
||||
// case is allowed to be lossy: the address text can be
|
||||
// identical across cities.
|
||||
st.WrongCity++
|
||||
rep.Wrong++
|
||||
rep.WrongExamples = appendExample(rep.WrongExamples, c, res.Address)
|
||||
case !c.RequireExact:
|
||||
// Missing information resolved to a different real address:
|
||||
// expected, not a safety violation.
|
||||
@@ -111,14 +124,17 @@ func Run(resolver *match.Resolver, cases []Case) Report {
|
||||
return rep
|
||||
}
|
||||
|
||||
// sameUnit compares house units, ignoring the structure component.
|
||||
// sameUnit compares house units, including the structure: two buildings that
|
||||
// differ only by "стр." are different addresses.
|
||||
func sameUnit(a, b address.ParsedAddress) bool {
|
||||
return a.House == b.House &&
|
||||
return a.CityKey == b.CityKey &&
|
||||
a.House == b.House &&
|
||||
a.HouseFrac == b.HouseFrac &&
|
||||
a.HouseLetter == b.HouseLetter &&
|
||||
a.Building == b.Building &&
|
||||
a.BuildingFrac == b.BuildingFrac &&
|
||||
a.BuildingLetter == b.BuildingLetter
|
||||
a.BuildingLetter == b.BuildingLetter &&
|
||||
a.Structure == b.Structure
|
||||
}
|
||||
|
||||
func appendExample(examples []string, c Case, got string) []string {
|
||||
@@ -185,16 +201,24 @@ func perturbations(e address.Entry, rng *rand.Rand) []Case {
|
||||
var out []Case
|
||||
add := func(query, category string, requireExact bool) {
|
||||
if query != "" && query != base {
|
||||
out = append(out, Case{Query: query, Expect: base, Category: category, RequireExact: requireExact})
|
||||
out = append(out, Case{
|
||||
Query: query, Expect: base, ExpectCity: e.City,
|
||||
Category: category, RequireExact: requireExact,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Baseline: the canonical address itself must always resolve exactly.
|
||||
out = append(out, Case{Query: base, Expect: base, Category: "baseline", RequireExact: true})
|
||||
out = append(out, Case{
|
||||
Query: base, Expect: base, ExpectCity: e.City,
|
||||
Category: "baseline", RequireExact: true,
|
||||
})
|
||||
|
||||
add(strings.ToLower(base), "case_lower", true)
|
||||
add(strings.ToUpper(base), "case_upper", true)
|
||||
add("г. Новосибирск, "+base, "city_prefix", true)
|
||||
if e.City != "" {
|
||||
add("г. "+e.City+", "+base, "city_prefix", true)
|
||||
}
|
||||
add(insertDom(base, e.House), "filler_dom", true)
|
||||
add(corruptPunctuation(base), "punctuation", true)
|
||||
add(addYo(base), "yo", true)
|
||||
@@ -212,7 +236,9 @@ func perturbations(e address.Entry, rng *rand.Rand) []Case {
|
||||
add(strings.Replace(base, "к"+e.Building, "корпус "+e.Building, 1), "corpus_synonym", true)
|
||||
}
|
||||
if hasStructure(base) {
|
||||
add(dropStructure(base), "structure_drop", true)
|
||||
// Dropping "стр. N" discards information: the base house may or may
|
||||
// not exist on its own, so a different real address is acceptable.
|
||||
add(dropStructure(base), "structure_drop", false)
|
||||
}
|
||||
if q := mutateLongestToken(base, rng); q != "" {
|
||||
add(q, "typo", true)
|
||||
|
||||
@@ -4,6 +4,9 @@ import (
|
||||
"math/rand"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
|
||||
)
|
||||
|
||||
// TestMoveTypeFront guards against the slice-aliasing regression that made the
|
||||
@@ -47,3 +50,37 @@ func TestMutateOnlyNameTokens(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunDetectsWrongCity guards the multi-city safety net: canonical address
|
||||
// text is identical across cities, so a found result must be checked against
|
||||
// the expected city, not only against street/house text.
|
||||
func TestRunDetectsWrongCity(t *testing.T) {
|
||||
parser := address.NewParser(address.NewDictionary([]address.City{{Name: "Бердск"}}), "")
|
||||
p := parser.ParseInCity("Ленина улица 5", "Бердск")
|
||||
entries := []address.Entry{{
|
||||
ID: "b1", Address: "Ленина улица 5",
|
||||
City: p.City, CityKey: p.CityKey,
|
||||
Ordinal: p.Ordinal, NameKey: p.NameKey, StreetKey: p.StreetKey, House: p.House,
|
||||
}}
|
||||
resolver := match.NewResolverWithOptions(
|
||||
match.NewIndex("test", entries, match.DefaultOptions()),
|
||||
nil,
|
||||
match.ResolverOptions{Parser: parser},
|
||||
)
|
||||
|
||||
cases := []Case{
|
||||
{Query: "Ленина 5", Expect: "Ленина улица 5", ExpectCity: "Новосибирск", Category: "wrong_city", RequireExact: true},
|
||||
{Query: "Ленина 5", Expect: "Ленина улица 5", ExpectCity: "Бердск", Category: "right_city", RequireExact: true},
|
||||
}
|
||||
rep := Run(resolver, cases)
|
||||
|
||||
if got := rep.Categories["wrong_city"]; got.WrongCity != 1 || got.Exact != 0 {
|
||||
t.Fatalf("wrong_city = %+v, want WrongCity=1 Exact=0", got)
|
||||
}
|
||||
if got := rep.Categories["right_city"]; got.Exact != 1 || got.WrongCity != 0 {
|
||||
t.Fatalf("right_city = %+v, want Exact=1 WrongCity=0", got)
|
||||
}
|
||||
if rep.Wrong != 1 {
|
||||
t.Fatalf("report Wrong = %d, want 1", rep.Wrong)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user