addrkit: quality evaluation harness + parser hardening
- add internal/eval: deterministic citizen-style perturbation harness (typos, case, word order, abbreviations, filler words, dropped units) - parser: strip all street-type tokens, filler words (город/дом/д/область), trailing apartment (кв N); handle letter-before-fraction and corpus letters - matching: token-aware typo-tolerant street similarity; wider ambiguity margin to surface near-collisions instead of guessing - seed: correct parse-rate accounting, structure-aware de-duplication - CLI: 'addrkit eval' command; README with measured results Measured on the real 73,936-row registry: 99.98% parsed, 0 wrong-house safety violations across 34k+ perturbed cases.
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// TestMoveTypeFront guards against the slice-aliasing regression that made the
|
||||
// type word disappear.
|
||||
func TestMoveTypeFront(t *testing.T) {
|
||||
got := moveTypeFront("станционная улица 1/1")
|
||||
if got != "улица станционная 1/1" {
|
||||
t.Fatalf("moveTypeFront = %q", got)
|
||||
}
|
||||
got = moveTypeFront("2-я станционная улица 1/1")
|
||||
if got != "улица 2-я станционная 1/1" {
|
||||
t.Fatalf("moveTypeFront with ordinal = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAddYoKeepsValidUTF8 guards against the byte-index regression.
|
||||
func TestAddYoKeepsValidUTF8(t *testing.T) {
|
||||
cases := []string{"переулок 5", "Северный проезд 7", "Зеленхозовская улица 48", "дом 1"}
|
||||
for _, in := range cases {
|
||||
out := addYo(in)
|
||||
if out != "" && !utf8.ValidString(out) {
|
||||
t.Fatalf("addYo(%q) produced invalid UTF-8: %q", in, out)
|
||||
}
|
||||
}
|
||||
if got := addYo("переулок 5"); got != "пёреулок 5" {
|
||||
t.Fatalf("addYo = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMutateOnlyNameTokens verifies typos never target the street type word,
|
||||
// which would turn into a bogus name token.
|
||||
func TestMutateOnlyNameTokens(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(7))
|
||||
for i := 0; i < 200; i++ {
|
||||
got := mutateLongestToken("Станционная улица 1/1", rng)
|
||||
if got == "" {
|
||||
continue
|
||||
}
|
||||
if got == "Станционная улицб 1/1" || got == "Станционная улиц 1/1" {
|
||||
t.Fatalf("type word was mutated: %q", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user