addrkit: quality evaluation harness + parser hardening
- add internal/eval: deterministic citizen-style perturbation harness (typos, case, word order, abbreviations, filler words, dropped units) - parser: strip all street-type tokens, filler words (город/дом/д/область), trailing apartment (кв N); handle letter-before-fraction and corpus letters - matching: token-aware typo-tolerant street similarity; wider ambiguity margin to surface near-collisions instead of guessing - seed: correct parse-rate accounting, structure-aware de-duplication - CLI: 'addrkit eval' command; README with measured results Measured on the real 73,936-row registry: 99.98% parsed, 0 wrong-house safety violations across 34k+ perturbed cases.
This commit is contained in:
@@ -20,6 +20,10 @@ var (
|
||||
|
||||
// "2-я", "10-й", "3-е"
|
||||
reOrdinalToken = regexp.MustCompile(`^(\d+)-[а-я]{1,3}$`)
|
||||
|
||||
// Trailing apartment part: " кв 5", " квартира 12", " кв. 7а". Removed
|
||||
// before house parsing so it cannot leak into the street name.
|
||||
reApartment = regexp.MustCompile(`(?:^|\s)(?:квартира|кв)\.?(?:\s+\d+[а-я]?)?\s*$`)
|
||||
)
|
||||
|
||||
// Parse decomposes a free-form address into its structured components. It is
|
||||
@@ -27,6 +31,8 @@ var (
|
||||
// part is best-effort and whose HasHouse is false.
|
||||
func Parse(raw string) ParsedAddress {
|
||||
normalized := NormalizeText(raw)
|
||||
// A trailing apartment ("кв 5") is not part of the address identity.
|
||||
normalized = strings.TrimSpace(reApartment.ReplaceAllString(normalized, ""))
|
||||
p := ParsedAddress{Raw: raw, Normalized: normalized}
|
||||
|
||||
rest := normalized
|
||||
@@ -69,28 +75,42 @@ func firstNonEmpty(values ...string) string {
|
||||
|
||||
// parseStreet fills the street-related fields of p from the leftover string
|
||||
// (the address with its house block removed).
|
||||
//
|
||||
// All recognised type tokens are removed from the name. This matters for
|
||||
// registry rows such as "Гусинобродский Тракт улица 21", where both "Тракт"
|
||||
// and "улица" are type-like: keeping either would mismatch a citizen who simply
|
||||
// says "Гусинобродский 21". The first type token becomes the canonical type.
|
||||
// If removing them would leave the name empty (a street literally called
|
||||
// "Набережная"), the type token is kept as the name.
|
||||
func parseStreet(p *ParsedAddress, s string) {
|
||||
tokens := strings.Fields(s)
|
||||
if len(tokens) == 0 {
|
||||
p.StreetKey = streetKey("", "")
|
||||
return
|
||||
}
|
||||
|
||||
name := make([]string, 0, len(tokens))
|
||||
typeFound := false
|
||||
streetType := ""
|
||||
|
||||
for _, tok := range tokens {
|
||||
// Only the first recognised type token is treated as the type; a
|
||||
// second occurrence is part of the name.
|
||||
if !typeFound {
|
||||
if t, ok := streetTypes[tok]; ok {
|
||||
p.StreetType = t
|
||||
typeFound = true
|
||||
continue
|
||||
// Filler tokens ("г", "город", "дом", "область", ...) never belong to
|
||||
// the street name.
|
||||
if fillerWords[tok] {
|
||||
continue
|
||||
}
|
||||
if t, ok := streetTypes[tok]; ok {
|
||||
if streetType == "" {
|
||||
streetType = t
|
||||
}
|
||||
continue
|
||||
}
|
||||
name = append(name, tok)
|
||||
}
|
||||
|
||||
p.StreetType = streetType
|
||||
if len(name) == 0 && streetType != "" {
|
||||
name = []string{streetType}
|
||||
}
|
||||
if len(name) == 0 {
|
||||
p.StreetKey = streetKey("", "")
|
||||
return
|
||||
}
|
||||
|
||||
name = extractOrdinal(p, name)
|
||||
p.NameDisplay = strings.Join(name, " ")
|
||||
|
||||
|
||||
@@ -77,6 +77,24 @@ func TestParseHouseBlock(t *testing.T) {
|
||||
wantOrdinal: "1", wantName: "ельцовка", wantType: "улица",
|
||||
wantHouse: "1", wantBLetter: "д", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "two type words in a registry name",
|
||||
raw: "Гусинобродский Тракт улица 21",
|
||||
wantName: "гусинобродский", wantType: "тракт",
|
||||
wantHouse: "21", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "name coincides with its type word",
|
||||
raw: "Набережная улица 5",
|
||||
wantName: "набережная", wantType: "набережная",
|
||||
wantHouse: "5", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "abbreviations dom and flat",
|
||||
raw: "ул. Кошурникова, д. 11/1 кв. 5",
|
||||
wantName: "кошурникова", wantType: "улица",
|
||||
wantHouse: "11", wantFrac: "1", wantHas: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -2,6 +2,18 @@ package address
|
||||
|
||||
import "strings"
|
||||
|
||||
// fillerWords are tokens that carry no street-name meaning and that people
|
||||
// routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
|
||||
// Dropping them makes the service tolerant of natural phrasing. District
|
||||
// adjectives are deliberately NOT dropped: they can legitimately be part of a
|
||||
// street name.
|
||||
var fillerWords = map[string]bool{
|
||||
"новосибирск": true, "нск": true, "нсо": true,
|
||||
"город": true, "г": true,
|
||||
"область": true, "обл": true,
|
||||
"район": true, "дом": true, "д": true,
|
||||
}
|
||||
|
||||
// streetTypes maps a surface form (including common abbreviations) to a single
|
||||
// canonical street type. The type is informational for matching (we strip it
|
||||
// from the key) but useful for disambiguation and display.
|
||||
@@ -31,8 +43,7 @@ var streetTypes = map[string]string{
|
||||
// ordinalWords maps spelled-out ordinals to their digit form. Only the forms
|
||||
// that actually occur in practice are listed; unknown words are treated as
|
||||
// part of the street name.
|
||||
var ordinalWords = map[string]string{
|
||||
"первый": "1", "первая": "1", "первое": "1",
|
||||
var ordinalWords = map[string]string{"первый": "1", "первая": "1", "первое": "1",
|
||||
"второй": "2", "вторая": "2", "второе": "2",
|
||||
"третий": "3", "третья": "3", "третье": "3",
|
||||
"четвертый": "4", "четвертая": "4", "четвертое": "4",
|
||||
|
||||
Reference in New Issue
Block a user