refactor!: make addrkit a multi-city address normalizer
- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
@@ -0,0 +1,198 @@
|
||||
package match
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
)
|
||||
|
||||
// staticRewriter returns a fixed list of variants, emulating the LLM adapter.
|
||||
type staticRewriter []string
|
||||
|
||||
func (s staticRewriter) Rewrite(context.Context, string) ([]string, error) { return s, nil }
|
||||
|
||||
// TestRewriteInheritsDetectedCity guards the multi-city LLM path: the embedded
|
||||
// prompt tells the model to strip administrative words, so a correct variant
|
||||
// usually has no city. It must inherit the city detected in the original query,
|
||||
// while a variant naming a different city must still be rejected.
|
||||
func TestRewriteInheritsDetectedCity(t *testing.T) {
|
||||
parser := address.NewParser(address.NewDictionary(matchCityDictionary()), "")
|
||||
entries := cityEntries(t, parser,
|
||||
[2]string{"Новосибирск", "Кошурникова улица 5"},
|
||||
[2]string{"Бердск", "Кошурникова улица 5"},
|
||||
)
|
||||
// A threshold above any typo score forces the deterministic path to miss.
|
||||
ix := NewIndex("test", entries, Options{FuzzyThreshold: 0.999, MaxCandidates: 5})
|
||||
|
||||
cityless := NewResolverWithOptions(ix, staticRewriter{"Кошурникова 5"}, ResolverOptions{Parser: parser})
|
||||
res := cityless.ResolveInCity(context.Background(), "нск кошурникиова 5", "")
|
||||
if res.Status != address.StatusFound || res.City != "Новосибирск" {
|
||||
t.Fatalf("cityless variant: %+v", res)
|
||||
}
|
||||
|
||||
conflict := NewResolverWithOptions(ix, staticRewriter{"Бердск, Кошурникова 5"}, ResolverOptions{Parser: parser})
|
||||
res = conflict.ResolveInCity(context.Background(), "нск кошурникиова 5", "")
|
||||
if res.Status == address.StatusFound {
|
||||
t.Fatalf("conflicting city variant was accepted: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func matchCityDictionary() []address.City {
|
||||
return []address.City{
|
||||
{Name: "Новосибирск", Aliases: []string{"нск"}},
|
||||
{Name: "Бердск"},
|
||||
}
|
||||
}
|
||||
|
||||
func cityEntries(t *testing.T, parser *address.Parser, rows ...[2]string) []address.Entry {
|
||||
t.Helper()
|
||||
entries := make([]address.Entry, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
p := parser.ParseInCity(row[1], row[0])
|
||||
entries = append(entries, address.Entry{
|
||||
ID: row[0] + "/" + row[1], Address: row[1],
|
||||
City: p.City, CityKey: p.CityKey,
|
||||
Ordinal: p.Ordinal, NameKey: p.NameKey, NameDisplay: p.NameDisplay,
|
||||
StreetType: p.StreetType, StreetKey: p.StreetKey,
|
||||
House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter,
|
||||
Building: p.Building, BuildingFrac: p.BuildingFrac, BuildingLetter: p.BuildingLetter,
|
||||
Structure: p.Structure,
|
||||
})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func TestIndexKeepsCitiesApart(t *testing.T) {
|
||||
parser := address.NewParser(address.NewDictionary(matchCityDictionary()), "")
|
||||
entries := cityEntries(t, parser,
|
||||
[2]string{"Новосибирск", "Ленина улица 5"},
|
||||
[2]string{"Бердск", "Ленина улица 5"},
|
||||
)
|
||||
resolver := NewResolverWithOptions(
|
||||
NewIndex("test", entries, DefaultOptions()),
|
||||
nil,
|
||||
ResolverOptions{Parser: parser},
|
||||
)
|
||||
|
||||
nsk := resolver.ResolveInCity(t.Context(), "Ленина 5", "Новосибирск")
|
||||
if nsk.Status != address.StatusFound || nsk.City != "Новосибирск" {
|
||||
t.Fatalf("Новосибирск: %+v", nsk)
|
||||
}
|
||||
berdsk := resolver.ResolveInCity(t.Context(), "Ленина 5", "Бердск")
|
||||
if berdsk.Status != address.StatusFound || berdsk.City != "Бердск" {
|
||||
t.Fatalf("Бердск: %+v", berdsk)
|
||||
}
|
||||
|
||||
// Without a city the resolver must not pick one: both streets are exact
|
||||
// matches in different cities.
|
||||
res := resolver.Resolve(t.Context(), "Ленина 5")
|
||||
if res.Status != address.StatusAmbiguous {
|
||||
t.Fatalf("cityless status = %q, want ambiguous (candidates: %+v)", res.Status, res.Candidates)
|
||||
}
|
||||
if len(res.Candidates) != 2 {
|
||||
t.Fatalf("candidates = %+v, want one per city", res.Candidates)
|
||||
}
|
||||
if res.City != "" {
|
||||
t.Fatalf("ambiguous cityless result must not report a city, got %q", res.City)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCitylessQueryReportsTheOnlyCity(t *testing.T) {
|
||||
parser := address.NewParser(address.NewDictionary(matchCityDictionary()), "")
|
||||
entries := cityEntries(t, parser, [2]string{"Новосибирск", "Ленина улица 5"})
|
||||
resolver := NewResolverWithOptions(
|
||||
NewIndex("test", entries, DefaultOptions()),
|
||||
nil,
|
||||
ResolverOptions{Parser: parser},
|
||||
)
|
||||
|
||||
res := resolver.Resolve(t.Context(), "Ленина 5")
|
||||
if res.Status != address.StatusFound || res.City != "Новосибирск" {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructureIsPartOfHouseIdentity(t *testing.T) {
|
||||
ix := buildIndex(t,
|
||||
"Станционная улица 1",
|
||||
"Станционная улица 1 стр. 2",
|
||||
)
|
||||
|
||||
found := ix.Resolve(address.Parse("станционная 1 стр. 2"))
|
||||
if found.Status != address.StatusFound || found.Address != "Станционная улица 1 стр. 2" {
|
||||
t.Fatalf("exact structure: %+v", found)
|
||||
}
|
||||
|
||||
// A structure that does not exist must never resolve to another building.
|
||||
wrong := ix.Resolve(address.Parse("станционная 1 стр. 3"))
|
||||
if wrong.Status == address.StatusFound {
|
||||
t.Fatalf("wrong structure resolved to %q", wrong.Address)
|
||||
}
|
||||
if wrong.Status != address.StatusAmbiguous {
|
||||
t.Fatalf("wrong structure status = %q, want ambiguous", wrong.Status)
|
||||
}
|
||||
|
||||
base := ix.Resolve(address.Parse("станционная 1"))
|
||||
if base.Status != address.StatusFound || base.Address != "Станционная улица 1" {
|
||||
t.Fatalf("base house: %+v", base)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexOrderIsDeterministic(t *testing.T) {
|
||||
addresses := []string{
|
||||
"Станционная улица 1/1",
|
||||
"Станционная улица 2/1",
|
||||
"Станционная улица 3/1 к1",
|
||||
"Станционная улица 3/1 к2",
|
||||
"Искитимская улица 5",
|
||||
}
|
||||
forward := buildEntries(t, addresses...)
|
||||
backward := buildEntries(t, reverse(addresses)...)
|
||||
|
||||
a := NewIndex("v", forward, DefaultOptions())
|
||||
b := NewIndex("v", backward, DefaultOptions())
|
||||
|
||||
for _, q := range []string{"станционная 3/1", "станционная 1/2", "искитимсая 5"} {
|
||||
ra := a.Resolve(address.Parse(q))
|
||||
rb := b.Resolve(address.Parse(q))
|
||||
if !reflect.DeepEqual(ra, rb) {
|
||||
t.Fatalf("%q: results differ\n a=%+v\n b=%+v", q, ra, rb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongTokenDoesNotMatch(t *testing.T) {
|
||||
ix := buildIndex(t, "Станционная улица 1")
|
||||
res := ix.Resolve(address.Parse(strings.Repeat("а", 4000)))
|
||||
if res.Status != address.StatusNotFound {
|
||||
t.Fatalf("status = %q, want not_found", res.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func buildEntries(t *testing.T, addresses ...string) []address.Entry {
|
||||
t.Helper()
|
||||
entries := make([]address.Entry, 0, len(addresses))
|
||||
for _, a := range addresses {
|
||||
p := address.Parse(a)
|
||||
entries = append(entries, address.Entry{
|
||||
ID: a, Address: a,
|
||||
Ordinal: p.Ordinal, NameKey: p.NameKey, NameDisplay: p.NameDisplay,
|
||||
StreetType: p.StreetType, StreetKey: p.StreetKey,
|
||||
House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter,
|
||||
Building: p.Building, BuildingFrac: p.BuildingFrac, BuildingLetter: p.BuildingLetter,
|
||||
Structure: p.Structure,
|
||||
})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func reverse(in []string) []string {
|
||||
out := make([]string, len(in))
|
||||
for i, s := range in {
|
||||
out[len(in)-1-i] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
+137
-32
@@ -9,6 +9,7 @@ package match
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
)
|
||||
@@ -16,16 +17,19 @@ import (
|
||||
// Candidate is one plausible match, returned alongside a non-"found" result so
|
||||
// the caller (or an operator) can see what the resolver was choosing between.
|
||||
type Candidate struct {
|
||||
Address string `json:"address"`
|
||||
District string `json:"district,omitempty"`
|
||||
House string `json:"house,omitempty"`
|
||||
Score float64 `json:"score"`
|
||||
Address string `json:"address"`
|
||||
City string `json:"city,omitempty"`
|
||||
District string `json:"district,omitempty"`
|
||||
House string `json:"house,omitempty"`
|
||||
Structure string `json:"structure,omitempty"`
|
||||
Score float64 `json:"score"`
|
||||
}
|
||||
|
||||
// Result is the outcome of a resolution.
|
||||
type Result struct {
|
||||
Status address.Status `json:"status"`
|
||||
Address string `json:"address,omitempty"`
|
||||
City string `json:"city,omitempty"`
|
||||
District string `json:"district,omitempty"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Candidates []Candidate `json:"candidates,omitempty"`
|
||||
@@ -59,10 +63,10 @@ func DefaultOptions() Options {
|
||||
}
|
||||
|
||||
type streetRef struct {
|
||||
key string
|
||||
ordinal string
|
||||
nameKey string
|
||||
streetType string
|
||||
key string
|
||||
cityKey string
|
||||
ordinal string
|
||||
nameKey string
|
||||
}
|
||||
|
||||
// Index is an immutable, in-memory registry index. It is safe for concurrent
|
||||
@@ -72,26 +76,46 @@ type Index struct {
|
||||
opts Options
|
||||
byKey map[string][]address.Entry
|
||||
streets []streetRef
|
||||
cities map[string]string // cityKey -> canonical display name
|
||||
size int
|
||||
}
|
||||
|
||||
// maxFuzzyNameRunes bounds the fuzzy scan. No real street name is longer than
|
||||
// this; without the bound a single huge token would make every Levenshtein
|
||||
// comparison walk a very long string and turn a request into a CPU sink.
|
||||
const maxFuzzyNameRunes = 96
|
||||
|
||||
// NewIndex builds an index from canonical entries. Entries sharing a StreetKey
|
||||
// are grouped; distinct streets are collected for typo-tolerant matching.
|
||||
//
|
||||
// Entries are sorted first, so candidate order is deterministic regardless of
|
||||
// the order in which the storage adapter returned them.
|
||||
func NewIndex(version string, entries []address.Entry, opts Options) *Index {
|
||||
sorted := append([]address.Entry(nil), entries...)
|
||||
sort.SliceStable(sorted, func(i, j int) bool { return entryLess(sorted[i], sorted[j]) })
|
||||
|
||||
ix := &Index{
|
||||
version: version,
|
||||
opts: opts,
|
||||
byKey: make(map[string][]address.Entry, len(entries)),
|
||||
byKey: make(map[string][]address.Entry, len(sorted)),
|
||||
cities: make(map[string]string),
|
||||
size: len(sorted),
|
||||
}
|
||||
seen := make(map[string]struct{}, len(entries))
|
||||
for _, e := range entries {
|
||||
seen := make(map[string]struct{}, len(sorted))
|
||||
for _, e := range sorted {
|
||||
ix.byKey[e.StreetKey] = append(ix.byKey[e.StreetKey], e)
|
||||
if e.CityKey != "" {
|
||||
if _, ok := ix.cities[e.CityKey]; !ok {
|
||||
ix.cities[e.CityKey] = e.City
|
||||
}
|
||||
}
|
||||
if _, ok := seen[e.StreetKey]; !ok {
|
||||
seen[e.StreetKey] = struct{}{}
|
||||
ix.streets = append(ix.streets, streetRef{
|
||||
key: e.StreetKey,
|
||||
ordinal: e.Ordinal,
|
||||
nameKey: e.NameKey,
|
||||
streetType: e.StreetType,
|
||||
key: e.StreetKey,
|
||||
cityKey: e.CityKey,
|
||||
ordinal: e.Ordinal,
|
||||
nameKey: e.NameKey,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -99,22 +123,57 @@ func NewIndex(version string, entries []address.Entry, opts Options) *Index {
|
||||
return ix
|
||||
}
|
||||
|
||||
// entryLess defines the canonical order of registry rows. Address strings are
|
||||
// the last meaningful tie-breaker; ID makes the order total even for duplicated
|
||||
// rows.
|
||||
func entryLess(a, b address.Entry) bool {
|
||||
switch {
|
||||
case a.CityKey != b.CityKey:
|
||||
return a.CityKey < b.CityKey
|
||||
case a.StreetKey != b.StreetKey:
|
||||
return a.StreetKey < b.StreetKey
|
||||
case a.House != b.House:
|
||||
return a.House < b.House
|
||||
case a.HouseFrac != b.HouseFrac:
|
||||
return a.HouseFrac < b.HouseFrac
|
||||
case a.HouseLetter != b.HouseLetter:
|
||||
return a.HouseLetter < b.HouseLetter
|
||||
case a.Building != b.Building:
|
||||
return a.Building < b.Building
|
||||
case a.BuildingFrac != b.BuildingFrac:
|
||||
return a.BuildingFrac < b.BuildingFrac
|
||||
case a.BuildingLetter != b.BuildingLetter:
|
||||
return a.BuildingLetter < b.BuildingLetter
|
||||
case a.Structure != b.Structure:
|
||||
return a.Structure < b.Structure
|
||||
case a.Address != b.Address:
|
||||
return a.Address < b.Address
|
||||
default:
|
||||
return a.ID < b.ID
|
||||
}
|
||||
}
|
||||
|
||||
// Version returns the data version the index was built from.
|
||||
func (ix *Index) Version() string { return ix.version }
|
||||
|
||||
// Size returns the number of indexed entries.
|
||||
func (ix *Index) Size() int {
|
||||
n := 0
|
||||
for _, v := range ix.byKey {
|
||||
n += len(v)
|
||||
func (ix *Index) Size() int { return ix.size }
|
||||
|
||||
// Cities returns the distinct canonical city names present in the index, in
|
||||
// deterministic order. A city-less index returns an empty slice.
|
||||
func (ix *Index) Cities() []string {
|
||||
out := make([]string, 0, len(ix.cities))
|
||||
for _, name := range ix.cities {
|
||||
out = append(out, name)
|
||||
}
|
||||
return n
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Resolve resolves a parsed address. It never returns an address that is not
|
||||
// present in the index.
|
||||
func (ix *Index) Resolve(p address.ParsedAddress) Result {
|
||||
res := Result{DataVersion: ix.version}
|
||||
res := Result{DataVersion: ix.version, City: p.City}
|
||||
|
||||
entries := ix.byKey[p.StreetKey]
|
||||
similarity := 1.0
|
||||
@@ -133,6 +192,11 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
|
||||
res.Status = address.StatusNotFound
|
||||
return res
|
||||
}
|
||||
if res.City == "" {
|
||||
// A city-less query may still match exactly one city; report it so the
|
||||
// caller does not have to re-derive the city from the candidates.
|
||||
res.City = commonCity(entries)
|
||||
}
|
||||
|
||||
if !p.HasHouse {
|
||||
// Street is known but the caller did not supply a house: safe but
|
||||
@@ -148,6 +212,7 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
|
||||
case 1:
|
||||
res.Status = address.StatusFound
|
||||
res.Address = matched[0].Address
|
||||
res.City = matched[0].City
|
||||
res.District = matched[0].District
|
||||
res.Confidence = clamp01(similarity)
|
||||
return res
|
||||
@@ -162,7 +227,14 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
|
||||
}
|
||||
res.Status = address.StatusPartial
|
||||
res.Confidence = clamp01(similarity * 0.9)
|
||||
res.Candidates = candidates(entries, similarity, ix.opts.MaxCandidates)
|
||||
// Prefer houses that share the number/fraction/letter: they are the
|
||||
// plausible corrections (a different corpus or structure), while the
|
||||
// rest of the street is noise.
|
||||
related := matchHouseUnitAll(entries, p)
|
||||
if len(related) == 0 {
|
||||
related = entries
|
||||
}
|
||||
res.Candidates = candidates(related, similarity, ix.opts.MaxCandidates)
|
||||
return res
|
||||
default:
|
||||
res.Status = address.StatusAmbiguous
|
||||
@@ -176,12 +248,16 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
|
||||
// the best street when the match is unambiguous, or the list of competing
|
||||
// street keys when several are within the ambiguity margin.
|
||||
func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry, similarity float64, ambiguousKeys []string) {
|
||||
if p.NameKey == "" {
|
||||
if p.NameKey == "" || utf8.RuneCountInString(p.NameKey) > maxFuzzyNameRunes {
|
||||
return nil, 0, nil
|
||||
}
|
||||
best := 0.0
|
||||
bestKey := ""
|
||||
for _, s := range ix.streets {
|
||||
// A query that names a city may only match streets in that city.
|
||||
if p.CityKey != "" && s.cityKey != p.CityKey {
|
||||
continue
|
||||
}
|
||||
// If the query carries an ordinal, the candidate's ordinal must match:
|
||||
// a user who says "1-й Северный" does not mean the plain "Северная".
|
||||
if !ordinalCompatible(p.Ordinal, s.ordinal) {
|
||||
@@ -197,6 +273,9 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
|
||||
}
|
||||
|
||||
for _, s := range ix.streets {
|
||||
if p.CityKey != "" && s.cityKey != p.CityKey {
|
||||
continue
|
||||
}
|
||||
if !ordinalCompatible(p.Ordinal, s.ordinal) {
|
||||
continue
|
||||
}
|
||||
@@ -232,10 +311,12 @@ func (ix *Index) representatives(keys []string, score float64) []Candidate {
|
||||
}
|
||||
e := es[0]
|
||||
out = append(out, Candidate{
|
||||
Address: e.Address,
|
||||
District: e.District,
|
||||
House: e.House,
|
||||
Score: clamp01(score),
|
||||
Address: e.Address,
|
||||
City: e.City,
|
||||
District: e.District,
|
||||
House: e.House,
|
||||
Structure: e.Structure,
|
||||
Score: clamp01(score),
|
||||
})
|
||||
if len(out) == ix.opts.MaxCandidates {
|
||||
break
|
||||
@@ -275,13 +356,16 @@ func sameHouseSeveral(entries []address.Entry, p address.ParsedAddress) bool {
|
||||
|
||||
// houseUnitEqual compares the parts of a house unit. Unspecified (empty) parts
|
||||
// must also be empty on the entry side, so "60/1" never matches "60/1 к148".
|
||||
// The structure ("стр. 2") is part of the identity: a caller who explicitly
|
||||
// asked for a different structure must not be handed another building.
|
||||
func houseUnitEqual(e address.Entry, p address.ParsedAddress) bool {
|
||||
return e.House == p.House &&
|
||||
e.HouseFrac == p.HouseFrac &&
|
||||
e.HouseLetter == p.HouseLetter &&
|
||||
e.Building == p.Building &&
|
||||
e.BuildingFrac == p.BuildingFrac &&
|
||||
e.BuildingLetter == p.BuildingLetter
|
||||
e.BuildingLetter == p.BuildingLetter &&
|
||||
e.Structure == p.Structure
|
||||
}
|
||||
|
||||
func candidates(entries []address.Entry, score float64, limit int) []Candidate {
|
||||
@@ -291,10 +375,12 @@ func candidates(entries []address.Entry, score float64, limit int) []Candidate {
|
||||
out := make([]Candidate, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
out = append(out, Candidate{
|
||||
Address: e.Address,
|
||||
District: e.District,
|
||||
House: e.House,
|
||||
Score: clamp01(score),
|
||||
Address: e.Address,
|
||||
City: e.City,
|
||||
District: e.District,
|
||||
House: e.House,
|
||||
Structure: e.Structure,
|
||||
Score: clamp01(score),
|
||||
})
|
||||
if len(out) == limit {
|
||||
break
|
||||
@@ -312,3 +398,22 @@ func clamp01(v float64) float64 {
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// commonCity returns the city shared by all entries, or "" when they span
|
||||
// cities (or carry no city at all).
|
||||
func commonCity(entries []address.Entry) string {
|
||||
key, name := "", ""
|
||||
for _, e := range entries {
|
||||
if e.CityKey == "" {
|
||||
continue
|
||||
}
|
||||
if key == "" {
|
||||
key, name = e.CityKey, e.City
|
||||
continue
|
||||
}
|
||||
if e.CityKey != key {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package match
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// levenshtein returns the edit distance between two strings, measured in
|
||||
// runes. It uses the rolling-row formulation: O(min(n,m)) memory.
|
||||
@@ -102,18 +105,30 @@ func min3(a, b, c int) int {
|
||||
return m
|
||||
}
|
||||
|
||||
// maxTokenRunes is the longest token the fuzzy matcher will compare. Real
|
||||
// street-name tokens are far shorter; anything longer is noise or abuse.
|
||||
const maxTokenRunes = 64
|
||||
|
||||
// tokenScore scores two name tokens. A single edit is tolerated for tokens of
|
||||
// length >= 4 and two edits for length >= 8; otherwise it falls back to the
|
||||
// plain ratio. Short names (<= 3 chars) are intentionally not fuzzy-matched:
|
||||
// one edit is too large a fraction of the word to be safe.
|
||||
//
|
||||
// Tokens longer than maxTokenRunes are never similar to a real street name.
|
||||
// Rejecting them up front keeps a pathological input from turning one request
|
||||
// into millions of Levenshtein cell updates.
|
||||
func tokenScore(a, b string) float64 {
|
||||
if a == b {
|
||||
return 1
|
||||
}
|
||||
ra, rb := utf8.RuneCountInString(a), utf8.RuneCountInString(b)
|
||||
if ra == 0 || rb == 0 || ra > maxTokenRunes || rb > maxTokenRunes {
|
||||
return 0
|
||||
}
|
||||
d := levenshtein(a, b)
|
||||
minLen := len([]rune(a))
|
||||
if lb := len([]rune(b)); lb < minLen {
|
||||
minLen = lb
|
||||
minLen := ra
|
||||
if rb < minLen {
|
||||
minLen = rb
|
||||
}
|
||||
switch {
|
||||
case d == 1 && minLen >= 4:
|
||||
|
||||
@@ -20,6 +20,25 @@ type Rewriter interface {
|
||||
Rewrite(ctx context.Context, raw string) ([]string, error)
|
||||
}
|
||||
|
||||
// Parser is the address parser used by the resolver. It is defined here
|
||||
// (consumer side) so the resolver can be tested with a stub and so a
|
||||
// deployment can plug in its own dictionary.
|
||||
type Parser interface {
|
||||
Parse(raw string) address.ParsedAddress
|
||||
ParseInCity(raw, city string) address.ParsedAddress
|
||||
MatchCity(query string) (address.City, bool)
|
||||
}
|
||||
|
||||
// ResolverOptions tunes optional resolver behaviour.
|
||||
type ResolverOptions struct {
|
||||
// Parser defaults to address.DefaultParser().
|
||||
Parser Parser
|
||||
// OnRewriteError, when set, observes failures of the optional rewriter.
|
||||
// The resolver always keeps the deterministic result, but operators still
|
||||
// need to know that the fallback is broken.
|
||||
OnRewriteError func(error)
|
||||
}
|
||||
|
||||
// Resolver wraps an Index with an optional Rewriter.
|
||||
//
|
||||
// The deterministic path is always tried first. The rewriter is consulted only
|
||||
@@ -27,13 +46,30 @@ type Rewriter interface {
|
||||
// suggestions must pass compatibleVariant before being considered, so the
|
||||
// model cannot substitute a different street or corrupt the house unit.
|
||||
type Resolver struct {
|
||||
index *Index
|
||||
rewriter Rewriter
|
||||
index *Index
|
||||
rewriter Rewriter
|
||||
parser Parser
|
||||
onRewriteError func(error)
|
||||
}
|
||||
|
||||
// NewResolver builds a resolver. rewriter may be nil to disable rewriting.
|
||||
// NewResolver builds a resolver with the default parser and no rewriter
|
||||
// diagnostics.
|
||||
func NewResolver(index *Index, rewriter Rewriter) *Resolver {
|
||||
return &Resolver{index: index, rewriter: rewriter}
|
||||
return NewResolverWithOptions(index, rewriter, ResolverOptions{})
|
||||
}
|
||||
|
||||
// NewResolverWithOptions builds a resolver with explicit options.
|
||||
func NewResolverWithOptions(index *Index, rewriter Rewriter, opts ResolverOptions) *Resolver {
|
||||
parser := opts.Parser
|
||||
if parser == nil {
|
||||
parser = address.DefaultParser()
|
||||
}
|
||||
return &Resolver{
|
||||
index: index,
|
||||
rewriter: rewriter,
|
||||
parser: parser,
|
||||
onRewriteError: opts.OnRewriteError,
|
||||
}
|
||||
}
|
||||
|
||||
// Version returns the current data version.
|
||||
@@ -42,9 +78,35 @@ func (r *Resolver) Version() string { return r.index.Version() }
|
||||
// Ready reports whether the resolver has a usable index.
|
||||
func (r *Resolver) Ready() bool { return r.index != nil && r.index.Size() > 0 }
|
||||
|
||||
// Cities returns the canonical city names present in the index.
|
||||
func (r *Resolver) Cities() []string {
|
||||
if r.index == nil {
|
||||
return nil
|
||||
}
|
||||
return r.index.Cities()
|
||||
}
|
||||
|
||||
// MatchCity reports whether query is a city name/alias and returns the
|
||||
// canonical name. It uses the resolver's parser dictionary, not the index, so
|
||||
// it also recognizes aliases for cities that have no rows yet.
|
||||
func (r *Resolver) MatchCity(query string) (string, bool) {
|
||||
city, ok := r.parser.MatchCity(query)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return city.Name, true
|
||||
}
|
||||
|
||||
// Resolve parses raw and resolves it, consulting the rewriter when enabled.
|
||||
func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
|
||||
parsed := address.Parse(raw)
|
||||
return r.ResolveInCity(ctx, raw, "")
|
||||
}
|
||||
|
||||
// ResolveInCity is Resolve with an explicit city hint. The hint overrides a
|
||||
// city found in the address text and lets clients pass the city as a separate
|
||||
// field.
|
||||
func (r *Resolver) ResolveInCity(ctx context.Context, raw, city string) Result {
|
||||
parsed := r.parse(raw, city)
|
||||
res := r.index.Resolve(parsed)
|
||||
|
||||
if res.Status == address.StatusFound || r.rewriter == nil {
|
||||
@@ -55,13 +117,27 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
|
||||
}
|
||||
|
||||
alternatives, err := r.rewriter.Rewrite(ctx, raw)
|
||||
if err != nil || len(alternatives) == 0 {
|
||||
if err != nil {
|
||||
if r.onRewriteError != nil {
|
||||
r.onRewriteError(err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
if len(alternatives) == 0 {
|
||||
return res
|
||||
}
|
||||
|
||||
best := res
|
||||
for _, alt := range alternatives {
|
||||
variant := address.Parse(alt)
|
||||
variant := r.parse(alt, city)
|
||||
// The embedded prompt asks the model to strip administrative words
|
||||
// (including the city), so a correct variant often has no city. It
|
||||
// inherits the original's city; only a conflicting explicit city is a
|
||||
// disqualification. Without this, the rewriter silently contributes
|
||||
// nothing in multi-city deployments (where default_city is empty).
|
||||
if variant.CityKey == "" && parsed.CityKey != "" {
|
||||
variant = r.parser.ParseInCity(alt, parsed.City)
|
||||
}
|
||||
// The model may fix the spelling of the street, nothing else.
|
||||
if !compatibleVariant(parsed, variant) {
|
||||
continue
|
||||
@@ -74,6 +150,13 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
|
||||
return best
|
||||
}
|
||||
|
||||
func (r *Resolver) parse(raw, city string) address.ParsedAddress {
|
||||
if strings.TrimSpace(city) != "" {
|
||||
return r.parser.ParseInCity(raw, city)
|
||||
}
|
||||
return r.parser.Parse(raw)
|
||||
}
|
||||
|
||||
// compatibleVariant reports whether an LLM rewrite may be considered. A rewrite
|
||||
// is acceptable only when it:
|
||||
// - keeps the exact house unit (number, fraction, letter, corpus, structure
|
||||
@@ -82,6 +165,9 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
|
||||
// - has the same words in the street name, each within a small edit distance
|
||||
// — so a different street can never be substituted.
|
||||
func compatibleVariant(orig, variant address.ParsedAddress) bool {
|
||||
if orig.CityKey != variant.CityKey {
|
||||
return false
|
||||
}
|
||||
if !sameHouseUnit(orig, variant) {
|
||||
return false
|
||||
}
|
||||
@@ -108,9 +194,13 @@ func nearToken(a, b string) bool {
|
||||
if a == b {
|
||||
return true
|
||||
}
|
||||
minLen := len([]rune(a))
|
||||
if lb := len([]rune(b)); lb < minLen {
|
||||
minLen = lb
|
||||
ra, rb := len([]rune(a)), len([]rune(b))
|
||||
if ra == 0 || rb == 0 || ra > maxTokenRunes || rb > maxTokenRunes {
|
||||
return false
|
||||
}
|
||||
minLen := ra
|
||||
if rb < minLen {
|
||||
minLen = rb
|
||||
}
|
||||
d := damerau(a, b)
|
||||
switch {
|
||||
@@ -129,7 +219,8 @@ func sameHouseUnit(a, b address.ParsedAddress) bool {
|
||||
a.HouseLetter == b.HouseLetter &&
|
||||
a.Building == b.Building &&
|
||||
a.BuildingFrac == b.BuildingFrac &&
|
||||
a.BuildingLetter == b.BuildingLetter
|
||||
a.BuildingLetter == b.BuildingLetter &&
|
||||
a.Structure == b.Structure
|
||||
}
|
||||
|
||||
// better ranks results: a stronger status wins, then higher confidence.
|
||||
|
||||
@@ -42,6 +42,26 @@ func TestCompatibleVariantProtectsApartment(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompatibleVariantProtectsCityAndStructure makes sure a rewrite cannot
|
||||
// move the address to another city or another building.
|
||||
func TestCompatibleVariantProtectsCityAndStructure(t *testing.T) {
|
||||
parser := address.NewParser(address.NewDictionary([]address.City{
|
||||
{Name: "Новосибирск", Aliases: []string{"нск"}},
|
||||
{Name: "Бердск"},
|
||||
}), "")
|
||||
orig := parser.Parse("нск, Сухарная улица 95 стр. 1")
|
||||
|
||||
if !compatibleVariant(orig, parser.Parse("Новосибирск, Сухраная улица 95 стр. 1")) {
|
||||
t.Error("same-city spelling fix must be accepted")
|
||||
}
|
||||
if compatibleVariant(orig, parser.Parse("Бердск, Сухарная улица 95 стр. 1")) {
|
||||
t.Error("city change must be rejected")
|
||||
}
|
||||
if compatibleVariant(orig, parser.Parse("нск, Сухарная улица 95 стр. 2")) {
|
||||
t.Error("structure change must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDamerau(t *testing.T) {
|
||||
if got := damerau("гоглоя", "гоголя"); got != 1 {
|
||||
t.Errorf("transposition distance = %d, want 1", got)
|
||||
|
||||
Reference in New Issue
Block a user