addrkit: quality evaluation harness + parser hardening

- add internal/eval: deterministic citizen-style perturbation harness
  (typos, case, word order, abbreviations, filler words, dropped units)
- parser: strip all street-type tokens, filler words (город/дом/д/область),
  trailing apartment (кв N); handle letter-before-fraction and corpus letters
- matching: token-aware typo-tolerant street similarity; wider ambiguity
  margin to surface near-collisions instead of guessing
- seed: correct parse-rate accounting, structure-aware de-duplication
- CLI: 'addrkit eval' command; README with measured results

Measured on the real 73,936-row registry: 99.98% parsed, 0 wrong-house
safety violations across 34k+ perturbed cases.
This commit is contained in:
addrkit
2026-10-09 20:13:50 +07:00
parent 3be331bb65
commit 5cc3556ac5
11 changed files with 690 additions and 25 deletions
+61
View File
@@ -1,5 +1,7 @@
package match
import "strings"
// levenshtein returns the edit distance between two strings, measured in
// runes. It uses the rolling-row formulation: O(min(n,m)) memory.
//
@@ -60,3 +62,62 @@ func min3(a, b, c int) int {
}
return m
}
// tokenScore scores two name tokens. A single edit is tolerated for tokens of
// length >= 4 and two edits for length >= 8; otherwise it falls back to the
// plain ratio. Short names (<= 3 chars) are intentionally not fuzzy-matched:
// one edit is too large a fraction of the word to be safe.
func tokenScore(a, b string) float64 {
if a == b {
return 1
}
d := levenshtein(a, b)
minLen := len([]rune(a))
if lb := len([]rune(b)); lb < minLen {
minLen = lb
}
switch {
case d == 1 && minLen >= 4:
return 0.95
case d == 2 && minLen >= 8:
return 0.90
default:
return ratio(a, b)
}
}
// nameSimilarity compares two canonical name keys (space-joined, sorted
// tokens) in a token-aware, order-insensitive way. A typo in one word of a
// multi-word name therefore does not sink the whole match, and reordered words
// still score 1.
func nameSimilarity(a, b string) float64 {
if a == b {
return 1
}
ta, tb := strings.Fields(a), strings.Fields(b)
if len(ta) == 0 || len(tb) == 0 {
return 0
}
sum, n := 0.0, 0
for _, x := range ta {
best := 0.0
for _, y := range tb {
if s := tokenScore(x, y); s > best {
best = s
}
}
sum += best
n++
}
for _, y := range tb {
best := 0.0
for _, x := range ta {
if s := tokenScore(x, y); s > best {
best = s
}
}
sum += best
n++
}
return sum / float64(n)
}