addrkit: quality evaluation harness + parser hardening
- add internal/eval: deterministic citizen-style perturbation harness (typos, case, word order, abbreviations, filler words, dropped units) - parser: strip all street-type tokens, filler words (город/дом/д/область), trailing apartment (кв N); handle letter-before-fraction and corpus letters - matching: token-aware typo-tolerant street similarity; wider ambiguity margin to surface near-collisions instead of guessing - seed: correct parse-rate accounting, structure-aware de-duplication - CLI: 'addrkit eval' command; README with measured results Measured on the real 73,936-row registry: 99.98% parsed, 0 wrong-house safety violations across 34k+ perturbed cases.
This commit is contained in:
@@ -101,6 +101,14 @@ func Build(sources []Source) ([]address.Entry, Stats) {
|
||||
appendSample(&stats, src.Address)
|
||||
continue
|
||||
}
|
||||
// Count parse quality on every source row, before de-duplication.
|
||||
if p.HasHouse {
|
||||
stats.Parsed++
|
||||
} else {
|
||||
stats.Unparsed++
|
||||
appendSample(&stats, src.Address)
|
||||
}
|
||||
|
||||
entry := address.Entry{
|
||||
ID: src.ID,
|
||||
Address: src.Address,
|
||||
@@ -133,12 +141,6 @@ func Build(sources []Source) ([]address.Entry, Stats) {
|
||||
|
||||
entries = append(entries, entry)
|
||||
streets[p.StreetKey] = struct{}{}
|
||||
if p.HasHouse {
|
||||
stats.Parsed++
|
||||
} else {
|
||||
stats.Unparsed++
|
||||
appendSample(&stats, src.Address)
|
||||
}
|
||||
}
|
||||
stats.DistinctStreets = len(streets)
|
||||
sort.Strings(stats.UnparsedSample)
|
||||
|
||||
Reference in New Issue
Block a user