addrkit: quality evaluation harness + parser hardening
- add internal/eval: deterministic citizen-style perturbation harness (typos, case, word order, abbreviations, filler words, dropped units) - parser: strip all street-type tokens, filler words (город/дом/д/область), trailing apartment (кв N); handle letter-before-fraction and corpus letters - matching: token-aware typo-tolerant street similarity; wider ambiguity margin to surface near-collisions instead of guessing - seed: correct parse-rate accounting, structure-aware de-duplication - CLI: 'addrkit eval' command; README with measured results Measured on the real 73,936-row registry: 99.98% parsed, 0 wrong-house safety violations across 34k+ perturbed cases.
This commit is contained in:
addrkit
@@ -0,0 +1,423 @@
|
||||
// Package eval is a reproducible, offline evaluation harness.
|
||||
//
|
||||
// It takes canonical registry entries, generates "citizen-style" variations
|
||||
// (typos, reordered words, abbreviations, dropped units, extra filler words)
|
||||
// and measures how the resolver copes. It exists so that reliability claims are
|
||||
// backed by numbers rather than intuition, and it is deterministic: the same
|
||||
// seed always produces the same cases.
|
||||
package eval
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
|
||||
)
|
||||
|
||||
// Case is one generated query plus its expectation.
|
||||
//
|
||||
// RequireExact marks variations that must still resolve to the same house unit
|
||||
// (case, word order, abbreviations, typos). Variations that legitimately drop
|
||||
// information (ordinal, house letter, fraction) do not require an exact match;
|
||||
// for them a different, real address is "lossy" rather than "wrong".
|
||||
type Case struct {
|
||||
Query string
|
||||
Expect string
|
||||
Category string
|
||||
RequireExact bool
|
||||
}
|
||||
|
||||
// CategoryStat aggregates results for one variation kind.
|
||||
type CategoryStat struct {
|
||||
Total int `json:"total"`
|
||||
Exact int `json:"exact"`
|
||||
Accuracy float64 `json:"accuracy"`
|
||||
Partial int `json:"partial"`
|
||||
Ambiguous int `json:"ambiguous"`
|
||||
NotFound int `json:"not_found"`
|
||||
WrongHouse int `json:"wrong_house"`
|
||||
WrongStreet int `json:"wrong_street"`
|
||||
Lossy int `json:"lossy"`
|
||||
}
|
||||
|
||||
// Report is the outcome of a run.
|
||||
type Report struct {
|
||||
Total int `json:"total"`
|
||||
Exact int `json:"exact"`
|
||||
ExactRate float64 `json:"exact_rate"`
|
||||
Wrong int `json:"wrong"` // safety violations on RequireExact cases
|
||||
WrongExamples []string `json:"wrong_examples,omitempty"`
|
||||
Mismatches []string `json:"mismatches,omitempty"`
|
||||
Categories map[string]CategoryStat `json:"categories"`
|
||||
}
|
||||
|
||||
// Run evaluates all cases against the index.
|
||||
func Run(ix *match.Index, cases []Case) Report {
|
||||
rep := Report{Categories: map[string]CategoryStat{}}
|
||||
for _, c := range cases {
|
||||
st := rep.Categories[c.Category]
|
||||
st.Total++
|
||||
rep.Total++
|
||||
|
||||
res := ix.Resolve(address.Parse(c.Query))
|
||||
exp := address.Parse(c.Expect)
|
||||
|
||||
exact := false
|
||||
switch res.Status {
|
||||
case address.StatusFound:
|
||||
rp := address.Parse(res.Address)
|
||||
equivalent := rp.StreetKey == exp.StreetKey && sameUnit(rp, exp)
|
||||
switch {
|
||||
case equivalent:
|
||||
st.Exact++
|
||||
rep.Exact++
|
||||
exact = true
|
||||
case !c.RequireExact:
|
||||
// Missing information resolved to a different real address:
|
||||
// expected, not a safety violation.
|
||||
st.Lossy++
|
||||
case rp.StreetKey == exp.StreetKey:
|
||||
st.WrongHouse++
|
||||
rep.Wrong++
|
||||
rep.WrongExamples = appendExample(rep.WrongExamples, c, res.Address)
|
||||
default:
|
||||
st.WrongStreet++
|
||||
rep.Wrong++
|
||||
rep.WrongExamples = appendExample(rep.WrongExamples, c, res.Address)
|
||||
}
|
||||
case address.StatusPartial:
|
||||
st.Partial++
|
||||
case address.StatusAmbiguous:
|
||||
st.Ambiguous++
|
||||
default:
|
||||
st.NotFound++
|
||||
}
|
||||
if !exact {
|
||||
rep.Mismatches = appendMismatch(rep.Mismatches, c, res)
|
||||
}
|
||||
if st.Total > 0 {
|
||||
st.Accuracy = float64(st.Exact) / float64(st.Total)
|
||||
}
|
||||
rep.Categories[c.Category] = st
|
||||
}
|
||||
if rep.Total > 0 {
|
||||
rep.ExactRate = float64(rep.Exact) / float64(rep.Total)
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
// sameUnit compares house units, ignoring the structure component.
|
||||
func sameUnit(a, b address.ParsedAddress) bool {
|
||||
return a.House == b.House &&
|
||||
a.HouseFrac == b.HouseFrac &&
|
||||
a.HouseLetter == b.HouseLetter &&
|
||||
a.Building == b.Building &&
|
||||
a.BuildingFrac == b.BuildingFrac &&
|
||||
a.BuildingLetter == b.BuildingLetter
|
||||
}
|
||||
|
||||
func appendExample(examples []string, c Case, got string) []string {
|
||||
const maxExamples = 25
|
||||
if len(examples) >= maxExamples {
|
||||
return examples
|
||||
}
|
||||
return append(examples, fmt.Sprintf("%s: %q -> %q (want %q)", c.Category, c.Query, got, c.Expect))
|
||||
}
|
||||
|
||||
// appendMismatch records non-exact outcomes (capped) for diagnosis. It is kept
|
||||
// separate from WrongExamples so safety violations stay prominent.
|
||||
func appendMismatch(examples []string, c Case, res match.Result) []string {
|
||||
const maxExamples = 60
|
||||
if len(examples) >= maxExamples {
|
||||
return examples
|
||||
}
|
||||
got := res.Address
|
||||
if got == "" && len(res.Candidates) > 0 {
|
||||
got = res.Candidates[0].Address
|
||||
}
|
||||
return append(examples, fmt.Sprintf("%s [%s]: %q -> %q (want %q)", c.Category, res.Status, c.Query, got, c.Expect))
|
||||
}
|
||||
|
||||
// Generate samples entries and produces perturbed cases from each.
|
||||
func Generate(entries []address.Entry, sample int, rng *rand.Rand) []Case {
|
||||
cand := make([]address.Entry, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.House != "" && e.NameKey != "" {
|
||||
cand = append(cand, e)
|
||||
}
|
||||
}
|
||||
rng.Shuffle(len(cand), func(i, j int) { cand[i], cand[j] = cand[j], cand[i] })
|
||||
if sample > 0 && sample < len(cand) {
|
||||
cand = cand[:sample]
|
||||
}
|
||||
|
||||
var cases []Case
|
||||
for _, e := range cand {
|
||||
cases = append(cases, perturbations(e, rng)...)
|
||||
}
|
||||
return cases
|
||||
}
|
||||
|
||||
var typeWords = []string{
|
||||
"улица", "переулок", "проезд", "проспект", "шоссе", "микрорайон",
|
||||
"бульвар", "площадь", "тупик", "спуск", "аллея", "набережная",
|
||||
"кольцо", "тракт", "поселок", "квартал", "территория", "дорога",
|
||||
}
|
||||
|
||||
var typeAbbrev = map[string]string{
|
||||
"улица": "ул.", "переулок": "пер.", "проезд": "пр.", "проспект": "пр-т",
|
||||
"шоссе": "ш.", "микрорайон": "мкр", "бульвар": "б-р", "площадь": "пл.",
|
||||
"тупик": "туп.", "набережная": "наб.", "поселок": "пос.",
|
||||
}
|
||||
|
||||
// perturbations returns the variations generated for one entry. Some are only
|
||||
// applicable when the corresponding component exists.
|
||||
func perturbations(e address.Entry, rng *rand.Rand) []Case {
|
||||
base := strings.TrimSpace(e.Address)
|
||||
if base == "" {
|
||||
return nil
|
||||
}
|
||||
var out []Case
|
||||
add := func(query, category string, requireExact bool) {
|
||||
if query != "" && query != base {
|
||||
out = append(out, Case{Query: query, Expect: base, Category: category, RequireExact: requireExact})
|
||||
}
|
||||
}
|
||||
|
||||
// Baseline: the canonical address itself must always resolve exactly.
|
||||
out = append(out, Case{Query: base, Expect: base, Category: "baseline", RequireExact: true})
|
||||
|
||||
add(strings.ToLower(base), "case_lower", true)
|
||||
add(strings.ToUpper(base), "case_upper", true)
|
||||
add("г. Новосибирск, "+base, "city_prefix", true)
|
||||
add(insertDom(base, e.House), "filler_dom", true)
|
||||
add(corruptPunctuation(base), "punctuation", true)
|
||||
add(addYo(base), "yo", true)
|
||||
|
||||
if q, ok := replaceType(base, func(t string) string { return "" }); ok {
|
||||
add(q, "type_removed", true)
|
||||
}
|
||||
if q, ok := replaceType(base, func(t string) string { return typeAbbrev[t] }); ok {
|
||||
add(q, "type_abbrev", true)
|
||||
}
|
||||
if q, ok := replaceType(base, func(t string) string { return t }); ok {
|
||||
add(moveTypeFront(q), "type_front", true)
|
||||
}
|
||||
if e.Building != "" {
|
||||
add(strings.Replace(base, "к"+e.Building, "корпус "+e.Building, 1), "corpus_synonym", true)
|
||||
}
|
||||
if hasStructure(base) {
|
||||
add(dropStructure(base), "structure_drop", true)
|
||||
}
|
||||
if q := mutateLongestToken(base, rng); q != "" {
|
||||
add(q, "typo", true)
|
||||
}
|
||||
|
||||
// Information-losing variations: not required to be exact.
|
||||
if e.Ordinal != "" {
|
||||
add(dropOrdinal(base), "drop_ordinal", false)
|
||||
}
|
||||
if e.HouseLetter != "" {
|
||||
add(strings.Replace(base, e.House+e.HouseLetter, e.House, 1), "drop_letter", false)
|
||||
}
|
||||
if e.HouseFrac != "" {
|
||||
add(strings.Replace(base, e.House+"/"+e.HouseFrac, e.House, 1), "drop_fraction", false)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// replaceType finds the first street-type token and rewrites it via fn. When fn
|
||||
// returns "" the token is removed. The bool reports whether a type was found.
|
||||
func replaceType(s string, fn func(t string) string) (string, bool) {
|
||||
tokens := strings.Fields(s)
|
||||
for i, tok := range tokens {
|
||||
low := strings.Trim(strings.ToLower(tok), ".")
|
||||
for _, t := range typeWords {
|
||||
if low == t {
|
||||
repl := fn(t)
|
||||
if repl == "" {
|
||||
tokens = append(tokens[:i], tokens[i+1:]...)
|
||||
} else {
|
||||
tokens[i] = repl
|
||||
}
|
||||
return strings.Join(tokens, " "), true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// moveTypeFront moves the (already present) type word to the front of the
|
||||
// street phrase, e.g. "станционная улица 1" -> "улица станционная 1".
|
||||
func moveTypeFront(s string) string {
|
||||
tokens := strings.Fields(s)
|
||||
for i, tok := range tokens {
|
||||
low := strings.Trim(strings.ToLower(tok), ".")
|
||||
for _, t := range typeWords {
|
||||
if low == t {
|
||||
// Build a fresh slice: appending to tokens[:i] would alias
|
||||
// and overwrite tokens[i] before it is captured.
|
||||
rest := make([]string, 0, len(tokens)-1)
|
||||
rest = append(rest, tokens[:i]...)
|
||||
rest = append(rest, tokens[i+1:]...)
|
||||
return strings.Join(append([]string{tok}, rest...), " ")
|
||||
}
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func insertDom(s, house string) string {
|
||||
if house == "" {
|
||||
return ""
|
||||
}
|
||||
idx := strings.LastIndex(s, " "+house)
|
||||
if idx < 0 {
|
||||
return ""
|
||||
}
|
||||
return s[:idx] + " дом" + s[idx:]
|
||||
}
|
||||
|
||||
func corruptPunctuation(s string) string {
|
||||
idx := strings.Index(s, " ")
|
||||
if idx < 0 {
|
||||
return ""
|
||||
}
|
||||
return s[:idx] + ", " + s[idx+1:]
|
||||
}
|
||||
|
||||
func addYo(s string) string {
|
||||
// Must operate on runes: a byte-index splice would cut a 2-byte UTF-8
|
||||
// Cyrillic letter in half.
|
||||
r := []rune(s)
|
||||
for i, ch := range r {
|
||||
if ch == 'е' {
|
||||
r[i] = 'ё'
|
||||
return string(r)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func hasStructure(s string) bool {
|
||||
return strings.Contains(s, " стр") || strings.Contains(s, " строение") || strings.Contains(s, " сооружение")
|
||||
}
|
||||
|
||||
func dropStructure(s string) string {
|
||||
for _, marker := range []string{" стр.", " строение", " сооружение", " стр"} {
|
||||
if idx := strings.Index(s, marker); idx >= 0 {
|
||||
return strings.TrimSpace(s[:idx])
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func dropOrdinal(s string) string {
|
||||
tokens := strings.Fields(s)
|
||||
if len(tokens) == 0 {
|
||||
return ""
|
||||
}
|
||||
first := tokens[0]
|
||||
if strings.Contains(first, "-") || isOrdinalWord(first) {
|
||||
return strings.Join(tokens[1:], " ")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func isOrdinalWord(s string) bool {
|
||||
switch strings.ToLower(s) {
|
||||
case "первый", "первая", "второй", "вторая", "третий", "третья",
|
||||
"четвертый", "четвертая", "пятый", "пятая", "шестой", "шестая",
|
||||
"седьмой", "седьмая", "восьмой", "восьмая", "девятый", "девятая",
|
||||
"десятый", "десятая":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// mutateLongestToken applies a single edit (delete, swap or substitute) to the
|
||||
// longest purely-Cyrillic *name* token, emulating a citizen typo. Type words
|
||||
// and numeric/ordinal tokens are excluded: corrupting "улица" must not be
|
||||
// treated as a typo in the street name.
|
||||
func mutateLongestToken(s string, rng *rand.Rand) string {
|
||||
tokens := strings.Fields(s)
|
||||
best := -1
|
||||
for i, tok := range tokens {
|
||||
if !isCyrillicWord(tok) || isTypeWord(tok) {
|
||||
continue
|
||||
}
|
||||
if best < 0 || len([]rune(tok)) > len([]rune(tokens[best])) {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return ""
|
||||
}
|
||||
r := []rune(tokens[best])
|
||||
if len(r) < 4 {
|
||||
return ""
|
||||
}
|
||||
switch rng.Intn(3) {
|
||||
case 0: // delete
|
||||
i := 1 + rng.Intn(len(r)-2)
|
||||
r = append(r[:i], r[i+1:]...)
|
||||
case 1: // swap adjacent
|
||||
i := 1 + rng.Intn(len(r)-2)
|
||||
r[i], r[i+1] = r[i+1], r[i]
|
||||
case 2: // substitute with a neighbouring Cyrillic letter
|
||||
i := 1 + rng.Intn(len(r)-2)
|
||||
r[i] = neighbour(r[i], rng)
|
||||
}
|
||||
tokens[best] = string(r)
|
||||
return strings.Join(tokens, " ")
|
||||
}
|
||||
|
||||
func isTypeWord(tok string) bool {
|
||||
low := strings.Trim(strings.ToLower(tok), ".")
|
||||
for _, t := range typeWords {
|
||||
if low == t {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isCyrillicWord(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range s {
|
||||
if !unicode.Is(unicode.Cyrillic, r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// neighbour returns a letter adjacent to r in the Russian alphabet.
|
||||
func neighbour(r rune, rng *rand.Rand) rune {
|
||||
if rng.Intn(2) == 0 {
|
||||
if r > 'а' {
|
||||
return r - 1
|
||||
}
|
||||
return r + 1
|
||||
}
|
||||
if r < 'я' {
|
||||
return r + 1
|
||||
}
|
||||
return r - 1
|
||||
}
|
||||
|
||||
// SortedCategories returns category names in a stable order (for reporting).
|
||||
func (r Report) SortedCategories() []string {
|
||||
names := make([]string, 0, len(r.Categories))
|
||||
for name := range r.Categories {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
Reference in New Issue
Block a user