addrkit: quality evaluation harness + parser hardening
- add internal/eval: deterministic citizen-style perturbation harness (typos, case, word order, abbreviations, filler words, dropped units) - parser: strip all street-type tokens, filler words (город/дом/д/область), trailing apartment (кв N); handle letter-before-fraction and corpus letters - matching: token-aware typo-tolerant street similarity; wider ambiguity margin to surface near-collisions instead of guessing - seed: correct parse-rate accounting, structure-aware de-duplication - CLI: 'addrkit eval' command; README with measured results Measured on the real 73,936-row registry: 99.98% parsed, 0 wrong-house safety violations across 34k+ perturbed cases.
This commit is contained in:
@@ -20,6 +20,10 @@ var (
|
||||
|
||||
// "2-я", "10-й", "3-е"
|
||||
reOrdinalToken = regexp.MustCompile(`^(\d+)-[а-я]{1,3}$`)
|
||||
|
||||
// Trailing apartment part: " кв 5", " квартира 12", " кв. 7а". Removed
|
||||
// before house parsing so it cannot leak into the street name.
|
||||
reApartment = regexp.MustCompile(`(?:^|\s)(?:квартира|кв)\.?(?:\s+\d+[а-я]?)?\s*$`)
|
||||
)
|
||||
|
||||
// Parse decomposes a free-form address into its structured components. It is
|
||||
@@ -27,6 +31,8 @@ var (
|
||||
// part is best-effort and whose HasHouse is false.
|
||||
func Parse(raw string) ParsedAddress {
|
||||
normalized := NormalizeText(raw)
|
||||
// A trailing apartment ("кв 5") is not part of the address identity.
|
||||
normalized = strings.TrimSpace(reApartment.ReplaceAllString(normalized, ""))
|
||||
p := ParsedAddress{Raw: raw, Normalized: normalized}
|
||||
|
||||
rest := normalized
|
||||
@@ -69,28 +75,42 @@ func firstNonEmpty(values ...string) string {
|
||||
|
||||
// parseStreet fills the street-related fields of p from the leftover string
|
||||
// (the address with its house block removed).
|
||||
//
|
||||
// All recognised type tokens are removed from the name. This matters for
|
||||
// registry rows such as "Гусинобродский Тракт улица 21", where both "Тракт"
|
||||
// and "улица" are type-like: keeping either would mismatch a citizen who simply
|
||||
// says "Гусинобродский 21". The first type token becomes the canonical type.
|
||||
// If removing them would leave the name empty (a street literally called
|
||||
// "Набережная"), the type token is kept as the name.
|
||||
func parseStreet(p *ParsedAddress, s string) {
|
||||
tokens := strings.Fields(s)
|
||||
if len(tokens) == 0 {
|
||||
p.StreetKey = streetKey("", "")
|
||||
return
|
||||
}
|
||||
|
||||
name := make([]string, 0, len(tokens))
|
||||
typeFound := false
|
||||
streetType := ""
|
||||
|
||||
for _, tok := range tokens {
|
||||
// Only the first recognised type token is treated as the type; a
|
||||
// second occurrence is part of the name.
|
||||
if !typeFound {
|
||||
if t, ok := streetTypes[tok]; ok {
|
||||
p.StreetType = t
|
||||
typeFound = true
|
||||
continue
|
||||
// Filler tokens ("г", "город", "дом", "область", ...) never belong to
|
||||
// the street name.
|
||||
if fillerWords[tok] {
|
||||
continue
|
||||
}
|
||||
if t, ok := streetTypes[tok]; ok {
|
||||
if streetType == "" {
|
||||
streetType = t
|
||||
}
|
||||
continue
|
||||
}
|
||||
name = append(name, tok)
|
||||
}
|
||||
|
||||
p.StreetType = streetType
|
||||
if len(name) == 0 && streetType != "" {
|
||||
name = []string{streetType}
|
||||
}
|
||||
if len(name) == 0 {
|
||||
p.StreetKey = streetKey("", "")
|
||||
return
|
||||
}
|
||||
|
||||
name = extractOrdinal(p, name)
|
||||
p.NameDisplay = strings.Join(name, " ")
|
||||
|
||||
|
||||
@@ -77,6 +77,24 @@ func TestParseHouseBlock(t *testing.T) {
|
||||
wantOrdinal: "1", wantName: "ельцовка", wantType: "улица",
|
||||
wantHouse: "1", wantBLetter: "д", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "two type words in a registry name",
|
||||
raw: "Гусинобродский Тракт улица 21",
|
||||
wantName: "гусинобродский", wantType: "тракт",
|
||||
wantHouse: "21", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "name coincides with its type word",
|
||||
raw: "Набережная улица 5",
|
||||
wantName: "набережная", wantType: "набережная",
|
||||
wantHouse: "5", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "abbreviations dom and flat",
|
||||
raw: "ул. Кошурникова, д. 11/1 кв. 5",
|
||||
wantName: "кошурникова", wantType: "улица",
|
||||
wantHouse: "11", wantFrac: "1", wantHas: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -2,6 +2,18 @@ package address
|
||||
|
||||
import "strings"
|
||||
|
||||
// fillerWords are tokens that carry no street-name meaning and that people
|
||||
// routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
|
||||
// Dropping them makes the service tolerant of natural phrasing. District
|
||||
// adjectives are deliberately NOT dropped: they can legitimately be part of a
|
||||
// street name.
|
||||
var fillerWords = map[string]bool{
|
||||
"новосибирск": true, "нск": true, "нсо": true,
|
||||
"город": true, "г": true,
|
||||
"область": true, "обл": true,
|
||||
"район": true, "дом": true, "д": true,
|
||||
}
|
||||
|
||||
// streetTypes maps a surface form (including common abbreviations) to a single
|
||||
// canonical street type. The type is informational for matching (we strip it
|
||||
// from the key) but useful for disambiguation and display.
|
||||
@@ -31,8 +43,7 @@ var streetTypes = map[string]string{
|
||||
// ordinalWords maps spelled-out ordinals to their digit form. Only the forms
|
||||
// that actually occur in practice are listed; unknown words are treated as
|
||||
// part of the street name.
|
||||
var ordinalWords = map[string]string{
|
||||
"первый": "1", "первая": "1", "первое": "1",
|
||||
var ordinalWords = map[string]string{"первый": "1", "первая": "1", "первое": "1",
|
||||
"второй": "2", "вторая": "2", "второе": "2",
|
||||
"третий": "3", "третья": "3", "третье": "3",
|
||||
"четвертый": "4", "четвертая": "4", "четвертое": "4",
|
||||
|
||||
@@ -45,10 +45,15 @@ type Options struct {
|
||||
}
|
||||
|
||||
// DefaultOptions returns conservative defaults.
|
||||
//
|
||||
// The ambiguity margin is deliberately generous relative to the token scores:
|
||||
// a one-edit match scores 0.95 and a two-edit match 0.90, so a 0.06 margin
|
||||
// makes near-collisions (e.g. "Елоховская" vs "Волховская") ambiguous rather
|
||||
// than silently picking the closer-but-wrong street.
|
||||
func DefaultOptions() Options {
|
||||
return Options{
|
||||
FuzzyThreshold: 0.82,
|
||||
AmbiguityMargin: 0.03,
|
||||
AmbiguityMargin: 0.06,
|
||||
MaxCandidates: 5,
|
||||
}
|
||||
}
|
||||
@@ -182,7 +187,7 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
|
||||
if p.Ordinal != "" && s.ordinal != "" && p.Ordinal != s.ordinal {
|
||||
continue
|
||||
}
|
||||
if sim := ratio(p.NameKey, s.nameKey); sim > best {
|
||||
if sim := nameSimilarity(p.NameKey, s.nameKey); sim > best {
|
||||
best = sim
|
||||
bestKey = s.key
|
||||
}
|
||||
@@ -195,7 +200,7 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
|
||||
if p.Ordinal != "" && s.ordinal != "" && p.Ordinal != s.ordinal {
|
||||
continue
|
||||
}
|
||||
if ratio(p.NameKey, s.nameKey) >= best-ix.opts.AmbiguityMargin {
|
||||
if nameSimilarity(p.NameKey, s.nameKey) >= best-ix.opts.AmbiguityMargin {
|
||||
ambiguousKeys = append(ambiguousKeys, s.key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package match
|
||||
|
||||
import "strings"
|
||||
|
||||
// levenshtein returns the edit distance between two strings, measured in
|
||||
// runes. It uses the rolling-row formulation: O(min(n,m)) memory.
|
||||
//
|
||||
@@ -60,3 +62,62 @@ func min3(a, b, c int) int {
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// tokenScore scores two name tokens. A single edit is tolerated for tokens of
|
||||
// length >= 4 and two edits for length >= 8; otherwise it falls back to the
|
||||
// plain ratio. Short names (<= 3 chars) are intentionally not fuzzy-matched:
|
||||
// one edit is too large a fraction of the word to be safe.
|
||||
func tokenScore(a, b string) float64 {
|
||||
if a == b {
|
||||
return 1
|
||||
}
|
||||
d := levenshtein(a, b)
|
||||
minLen := len([]rune(a))
|
||||
if lb := len([]rune(b)); lb < minLen {
|
||||
minLen = lb
|
||||
}
|
||||
switch {
|
||||
case d == 1 && minLen >= 4:
|
||||
return 0.95
|
||||
case d == 2 && minLen >= 8:
|
||||
return 0.90
|
||||
default:
|
||||
return ratio(a, b)
|
||||
}
|
||||
}
|
||||
|
||||
// nameSimilarity compares two canonical name keys (space-joined, sorted
|
||||
// tokens) in a token-aware, order-insensitive way. A typo in one word of a
|
||||
// multi-word name therefore does not sink the whole match, and reordered words
|
||||
// still score 1.
|
||||
func nameSimilarity(a, b string) float64 {
|
||||
if a == b {
|
||||
return 1
|
||||
}
|
||||
ta, tb := strings.Fields(a), strings.Fields(b)
|
||||
if len(ta) == 0 || len(tb) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum, n := 0.0, 0
|
||||
for _, x := range ta {
|
||||
best := 0.0
|
||||
for _, y := range tb {
|
||||
if s := tokenScore(x, y); s > best {
|
||||
best = s
|
||||
}
|
||||
}
|
||||
sum += best
|
||||
n++
|
||||
}
|
||||
for _, y := range tb {
|
||||
best := 0.0
|
||||
for _, x := range ta {
|
||||
if s := tokenScore(x, y); s > best {
|
||||
best = s
|
||||
}
|
||||
}
|
||||
sum += best
|
||||
n++
|
||||
}
|
||||
return sum / float64(n)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user