addrkit: quality evaluation harness + parser hardening
- add internal/eval: deterministic citizen-style perturbation harness (typos, case, word order, abbreviations, filler words, dropped units) - parser: strip all street-type tokens, filler words (город/дом/д/область), trailing apartment (кв N); handle letter-before-fraction and corpus letters - matching: token-aware typo-tolerant street similarity; wider ambiguity margin to surface near-collisions instead of guessing - seed: correct parse-rate accounting, structure-aware de-duplication - CLI: 'addrkit eval' command; README with measured results Measured on the real 73,936-row registry: 99.98% parsed, 0 wrong-house safety violations across 34k+ perturbed cases.
This commit is contained in:
@@ -45,10 +45,15 @@ type Options struct {
|
||||
}
|
||||
|
||||
// DefaultOptions returns conservative defaults.
|
||||
//
|
||||
// The ambiguity margin is deliberately generous relative to the token scores:
|
||||
// a one-edit match scores 0.95 and a two-edit match 0.90, so a 0.06 margin
|
||||
// makes near-collisions (e.g. "Елоховская" vs "Волховская") ambiguous rather
|
||||
// than silently picking the closer-but-wrong street.
|
||||
func DefaultOptions() Options {
|
||||
return Options{
|
||||
FuzzyThreshold: 0.82,
|
||||
AmbiguityMargin: 0.03,
|
||||
AmbiguityMargin: 0.06,
|
||||
MaxCandidates: 5,
|
||||
}
|
||||
}
|
||||
@@ -182,7 +187,7 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
|
||||
if p.Ordinal != "" && s.ordinal != "" && p.Ordinal != s.ordinal {
|
||||
continue
|
||||
}
|
||||
if sim := ratio(p.NameKey, s.nameKey); sim > best {
|
||||
if sim := nameSimilarity(p.NameKey, s.nameKey); sim > best {
|
||||
best = sim
|
||||
bestKey = s.key
|
||||
}
|
||||
@@ -195,7 +200,7 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
|
||||
if p.Ordinal != "" && s.ordinal != "" && p.Ordinal != s.ordinal {
|
||||
continue
|
||||
}
|
||||
if ratio(p.NameKey, s.nameKey) >= best-ix.opts.AmbiguityMargin {
|
||||
if nameSimilarity(p.NameKey, s.nameKey) >= best-ix.opts.AmbiguityMargin {
|
||||
ambiguousKeys = append(ambiguousKeys, s.key)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user