addrkit: quality evaluation harness + parser hardening

- add internal/eval: deterministic citizen-style perturbation harness
  (typos, case, word order, abbreviations, filler words, dropped units)
- parser: strip all street-type tokens, filler words (город/дом/д/область),
  trailing apartment (кв N); handle letter-before-fraction and corpus letters
- matching: token-aware typo-tolerant street similarity; wider ambiguity
  margin to surface near-collisions instead of guessing
- seed: correct parse-rate accounting, structure-aware de-duplication
- CLI: 'addrkit eval' command; README with measured results

Measured on the real 73,936-row registry: 99.98% parsed, 0 wrong-house
safety violations across 34k+ perturbed cases.
This commit is contained in:
Maksim Totmin
2026-10-09 21:40:46 +07:00
parent 7b152c7fb3
commit ba4705fc4f
11 changed files with 690 additions and 25 deletions
+49
View File
@@ -0,0 +1,49 @@
package eval
import (
"math/rand"
"testing"
"unicode/utf8"
)
// TestMoveTypeFront guards against the slice-aliasing regression that made the
// type word disappear.
func TestMoveTypeFront(t *testing.T) {
got := moveTypeFront("станционная улица 1/1")
if got != "улица станционная 1/1" {
t.Fatalf("moveTypeFront = %q", got)
}
got = moveTypeFront("2-я станционная улица 1/1")
if got != "улица 2-я станционная 1/1" {
t.Fatalf("moveTypeFront with ordinal = %q", got)
}
}
// TestAddYoKeepsValidUTF8 guards against the byte-index regression.
func TestAddYoKeepsValidUTF8(t *testing.T) {
cases := []string{"переулок 5", "Северный проезд 7", "Зеленхозовская улица 48", "дом 1"}
for _, in := range cases {
out := addYo(in)
if out != "" && !utf8.ValidString(out) {
t.Fatalf("addYo(%q) produced invalid UTF-8: %q", in, out)
}
}
if got := addYo("переулок 5"); got != "пёреулок 5" {
t.Fatalf("addYo = %q", got)
}
}
// TestMutateOnlyNameTokens verifies typos never target the street type word,
// which would turn into a bogus name token.
func TestMutateOnlyNameTokens(t *testing.T) {
rng := rand.New(rand.NewSource(7))
for i := 0; i < 200; i++ {
got := mutateLongestToken("Станционная улица 1/1", rng)
if got == "" {
continue
}
if got == "Станционная улицб 1/1" || got == "Станционная улиц 1/1" {
t.Fatalf("type word was mutated: %q", got)
}
}
}