Maksim Totmin
ba4705fc4f
- add internal/eval: deterministic citizen-style perturbation harness (typos, case, word order, abbreviations, filler words, dropped units) - parser: strip all street-type tokens, filler words (город/дом/д/область), trailing apartment (кв N); handle letter-before-fraction and corpus letters - matching: token-aware typo-tolerant street similarity; wider ambiguity margin to surface near-collisions instead of guessing - seed: correct parse-rate accounting, structure-aware de-duplication - CLI: 'addrkit eval' command; README with measured results Measured on the real 73,936-row registry: 99.98% parsed, 0 wrong-house safety violations across 34k+ perturbed cases.
124 lines
2.6 KiB
Go
124 lines
2.6 KiB
Go
package match
|
|
|
|
import "strings"
|
|
|
|
// levenshtein returns the edit distance between two strings, measured in
|
|
// runes. It uses the rolling-row formulation: O(min(n,m)) memory.
|
|
//
|
|
// The resolver uses it only to tolerate typos in the *street name*; house
|
|
// numbers are always compared exactly, because a wrong house is worse than a
|
|
// missing one.
|
|
func levenshtein(a, b string) int {
|
|
ra, rb := []rune(a), []rune(b)
|
|
if len(ra) == 0 {
|
|
return len(rb)
|
|
}
|
|
if len(rb) == 0 {
|
|
return len(ra)
|
|
}
|
|
prev := make([]int, len(rb)+1)
|
|
curr := make([]int, len(rb)+1)
|
|
for j := range prev {
|
|
prev[j] = j
|
|
}
|
|
for i := 1; i <= len(ra); i++ {
|
|
curr[0] = i
|
|
for j := 1; j <= len(rb); j++ {
|
|
cost := 1
|
|
if ra[i-1] == rb[j-1] {
|
|
cost = 0
|
|
}
|
|
curr[j] = min3(prev[j]+1, curr[j-1]+1, prev[j-1]+cost)
|
|
}
|
|
prev, curr = curr, prev
|
|
}
|
|
return prev[len(rb)]
|
|
}
|
|
|
|
// ratio returns a 0..1 similarity where 1 means identical. It is
|
|
// 1 - distance/max(len), which is stable and easy to reason about.
|
|
func ratio(a, b string) float64 {
|
|
if a == b {
|
|
return 1
|
|
}
|
|
la, lb := len([]rune(a)), len([]rune(b))
|
|
m := la
|
|
if lb > m {
|
|
m = lb
|
|
}
|
|
if m == 0 {
|
|
return 1
|
|
}
|
|
return 1 - float64(levenshtein(a, b))/float64(m)
|
|
}
|
|
|
|
func min3(a, b, c int) int {
|
|
m := a
|
|
if b < m {
|
|
m = b
|
|
}
|
|
if c < m {
|
|
m = c
|
|
}
|
|
return m
|
|
}
|
|
|
|
// tokenScore scores two name tokens. A single edit is tolerated for tokens of
|
|
// length >= 4 and two edits for length >= 8; otherwise it falls back to the
|
|
// plain ratio. Short names (<= 3 chars) are intentionally not fuzzy-matched:
|
|
// one edit is too large a fraction of the word to be safe.
|
|
func tokenScore(a, b string) float64 {
|
|
if a == b {
|
|
return 1
|
|
}
|
|
d := levenshtein(a, b)
|
|
minLen := len([]rune(a))
|
|
if lb := len([]rune(b)); lb < minLen {
|
|
minLen = lb
|
|
}
|
|
switch {
|
|
case d == 1 && minLen >= 4:
|
|
return 0.95
|
|
case d == 2 && minLen >= 8:
|
|
return 0.90
|
|
default:
|
|
return ratio(a, b)
|
|
}
|
|
}
|
|
|
|
// nameSimilarity compares two canonical name keys (space-joined, sorted
|
|
// tokens) in a token-aware, order-insensitive way. A typo in one word of a
|
|
// multi-word name therefore does not sink the whole match, and reordered words
|
|
// still score 1.
|
|
func nameSimilarity(a, b string) float64 {
|
|
if a == b {
|
|
return 1
|
|
}
|
|
ta, tb := strings.Fields(a), strings.Fields(b)
|
|
if len(ta) == 0 || len(tb) == 0 {
|
|
return 0
|
|
}
|
|
sum, n := 0.0, 0
|
|
for _, x := range ta {
|
|
best := 0.0
|
|
for _, y := range tb {
|
|
if s := tokenScore(x, y); s > best {
|
|
best = s
|
|
}
|
|
}
|
|
sum += best
|
|
n++
|
|
}
|
|
for _, y := range tb {
|
|
best := 0.0
|
|
for _, x := range ta {
|
|
if s := tokenScore(x, y); s > best {
|
|
best = s
|
|
}
|
|
}
|
|
sum += best
|
|
n++
|
|
}
|
|
return sum / float64(n)
|
|
}
|