refactor!: make addrkit a multi-city address normalizer
- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
@@ -1,6 +1,9 @@
|
||||
package match
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// levenshtein returns the edit distance between two strings, measured in
|
||||
// runes. It uses the rolling-row formulation: O(min(n,m)) memory.
|
||||
@@ -102,18 +105,30 @@ func min3(a, b, c int) int {
|
||||
return m
|
||||
}
|
||||
|
||||
// maxTokenRunes is the longest token the fuzzy matcher will compare. Real
|
||||
// street-name tokens are far shorter; anything longer is noise or abuse.
|
||||
const maxTokenRunes = 64
|
||||
|
||||
// tokenScore scores two name tokens. A single edit is tolerated for tokens of
|
||||
// length >= 4 and two edits for length >= 8; otherwise it falls back to the
|
||||
// plain ratio. Short names (<= 3 chars) are intentionally not fuzzy-matched:
|
||||
// one edit is too large a fraction of the word to be safe.
|
||||
//
|
||||
// Tokens longer than maxTokenRunes are never similar to a real street name.
|
||||
// Rejecting them up front keeps a pathological input from turning one request
|
||||
// into millions of Levenshtein cell updates.
|
||||
func tokenScore(a, b string) float64 {
|
||||
if a == b {
|
||||
return 1
|
||||
}
|
||||
ra, rb := utf8.RuneCountInString(a), utf8.RuneCountInString(b)
|
||||
if ra == 0 || rb == 0 || ra > maxTokenRunes || rb > maxTokenRunes {
|
||||
return 0
|
||||
}
|
||||
d := levenshtein(a, b)
|
||||
minLen := len([]rune(a))
|
||||
if lb := len([]rune(b)); lb < minLen {
|
||||
minLen = lb
|
||||
minLen := ra
|
||||
if rb < minLen {
|
||||
minLen = rb
|
||||
}
|
||||
switch {
|
||||
case d == 1 && minLen >= 4:
|
||||
|
||||
Reference in New Issue
Block a user