addrkit: quality evaluation harness + parser hardening
- add internal/eval: deterministic citizen-style perturbation harness (typos, case, word order, abbreviations, filler words, dropped units) - parser: strip all street-type tokens, filler words (город/дом/д/область), trailing apartment (кв N); handle letter-before-fraction and corpus letters - matching: token-aware typo-tolerant street similarity; wider ambiguity margin to surface near-collisions instead of guessing - seed: correct parse-rate accounting, structure-aware de-duplication - CLI: 'addrkit eval' command; README with measured results Measured on the real 73,936-row registry: 99.98% parsed, 0 wrong-house safety violations across 34k+ perturbed cases.
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
package match
|
||||
|
||||
import "strings"
|
||||
|
||||
// levenshtein returns the edit distance between two strings, measured in
|
||||
// runes. It uses the rolling-row formulation: O(min(n,m)) memory.
|
||||
//
|
||||
@@ -60,3 +62,62 @@ func min3(a, b, c int) int {
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// tokenScore scores two name tokens. A single edit is tolerated for tokens of
|
||||
// length >= 4 and two edits for length >= 8; otherwise it falls back to the
|
||||
// plain ratio. Short names (<= 3 chars) are intentionally not fuzzy-matched:
|
||||
// one edit is too large a fraction of the word to be safe.
|
||||
func tokenScore(a, b string) float64 {
|
||||
if a == b {
|
||||
return 1
|
||||
}
|
||||
d := levenshtein(a, b)
|
||||
minLen := len([]rune(a))
|
||||
if lb := len([]rune(b)); lb < minLen {
|
||||
minLen = lb
|
||||
}
|
||||
switch {
|
||||
case d == 1 && minLen >= 4:
|
||||
return 0.95
|
||||
case d == 2 && minLen >= 8:
|
||||
return 0.90
|
||||
default:
|
||||
return ratio(a, b)
|
||||
}
|
||||
}
|
||||
|
||||
// nameSimilarity compares two canonical name keys (space-joined, sorted
|
||||
// tokens) in a token-aware, order-insensitive way. A typo in one word of a
|
||||
// multi-word name therefore does not sink the whole match, and reordered words
|
||||
// still score 1.
|
||||
func nameSimilarity(a, b string) float64 {
|
||||
if a == b {
|
||||
return 1
|
||||
}
|
||||
ta, tb := strings.Fields(a), strings.Fields(b)
|
||||
if len(ta) == 0 || len(tb) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum, n := 0.0, 0
|
||||
for _, x := range ta {
|
||||
best := 0.0
|
||||
for _, y := range tb {
|
||||
if s := tokenScore(x, y); s > best {
|
||||
best = s
|
||||
}
|
||||
}
|
||||
sum += best
|
||||
n++
|
||||
}
|
||||
for _, y := range tb {
|
||||
best := 0.0
|
||||
for _, x := range ta {
|
||||
if s := tokenScore(x, y); s > best {
|
||||
best = s
|
||||
}
|
||||
}
|
||||
sum += best
|
||||
n++
|
||||
}
|
||||
return sum / float64(n)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user