refactor!: make addrkit a multi-city address normalizer
- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
package address
|
||||
|
||||
import "testing"
|
||||
|
||||
func testDictionary() *Dictionary {
|
||||
return NewDictionary([]City{
|
||||
{Name: "Новосибирск", Aliases: []string{"нск", "новосиб", "новосибирская область", "нсо"}},
|
||||
{Name: "Бердск"},
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseCityFromAliases(t *testing.T) {
|
||||
parser := NewParser(testDictionary(), "")
|
||||
|
||||
tests := []struct {
|
||||
raw string
|
||||
city string
|
||||
nameKey string
|
||||
}{
|
||||
{"нск, 2-я Станционная улица 1", "Новосибирск", "станционная"},
|
||||
{"НСК Ленина 5", "Новосибирск", "ленина"},
|
||||
{"Новосибирская область, Станционная 1", "Новосибирск", "станционная"},
|
||||
{"Бердск, Ленина 5", "Бердск", "ленина"},
|
||||
{"г. Новосибирск, ул. Ленина 5", "Новосибирск", "ленина"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := parser.Parse(tt.raw)
|
||||
if got.City != tt.city {
|
||||
t.Errorf("%q: City = %q, want %q", tt.raw, got.City, tt.city)
|
||||
}
|
||||
if got.NameKey != tt.nameKey {
|
||||
t.Errorf("%q: NameKey = %q, want %q", tt.raw, got.NameKey, tt.nameKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCityIsPartOfStreetKey(t *testing.T) {
|
||||
parser := NewParser(testDictionary(), "")
|
||||
|
||||
nsk := parser.Parse("Новосибирск, Ленина 5")
|
||||
berdsk := parser.Parse("Бердск, Ленина 5")
|
||||
if nsk.StreetKey == berdsk.StreetKey {
|
||||
t.Fatalf("same street key for different cities: %q", nsk.StreetKey)
|
||||
}
|
||||
if nsk.CityKey != "новосибирск" || berdsk.CityKey != "бердск" {
|
||||
t.Fatalf("city keys = %q / %q", nsk.CityKey, berdsk.CityKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDefaultCity(t *testing.T) {
|
||||
parser := NewParser(testDictionary(), "Новосибирск")
|
||||
got := parser.Parse("Ленина 5")
|
||||
if got.City != "Новосибирск" || got.CityKey != "новосибирск" {
|
||||
t.Fatalf("City = %q / %q", got.City, got.CityKey)
|
||||
}
|
||||
|
||||
// The default city is also recognised when the user spells it out, even
|
||||
// if the dictionary contains only aliases.
|
||||
explicit := NewParser(DefaultDictionary(), "Новосибирск").Parse("г. Новосибирск, Ленина 5")
|
||||
if explicit.City != "Новосибирск" || explicit.NameKey != "ленина" {
|
||||
t.Fatalf("explicit default city: %+v", explicit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInCityOverridesDetectedCity(t *testing.T) {
|
||||
parser := NewParser(testDictionary(), "")
|
||||
got := parser.ParseInCity("Бердск, Ленина 5", "Новосибирск")
|
||||
if got.City != "Новосибирск" {
|
||||
t.Fatalf("City = %q, want forced Новосибирск", got.City)
|
||||
}
|
||||
if got.NameKey != "ленина" {
|
||||
t.Fatalf("NameKey = %q", got.NameKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWithoutCityKeepsCityKeyEmpty(t *testing.T) {
|
||||
parser := NewParser(testDictionary(), "")
|
||||
got := parser.Parse("Ленина 5")
|
||||
if got.City != "" || got.CityKey != "" {
|
||||
t.Fatalf("City = %q / %q, want empty", got.City, got.CityKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeTextStillLeavesUnknownCityTokens(t *testing.T) {
|
||||
// A city that is not in the dictionary must not be silently dropped: the
|
||||
// resolver should fail rather than guess another city.
|
||||
parser := NewParser(testDictionary(), "")
|
||||
got := parser.Parse("Томск, Ленина 5")
|
||||
if got.City != "" {
|
||||
t.Fatalf("City = %q, want empty for unknown city", got.City)
|
||||
}
|
||||
if got.NameKey == "ленина" {
|
||||
t.Fatalf("unknown city token disappeared from the street name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchCity(t *testing.T) {
|
||||
dict := testDictionary()
|
||||
cases := []struct {
|
||||
query string
|
||||
want string
|
||||
ok bool
|
||||
}{
|
||||
{"нск", "Новосибирск", true},
|
||||
{"г. Новосибирск", "Новосибирск", true},
|
||||
{"Новосибирская область", "Новосибирск", true},
|
||||
{"Бердск", "Бердск", true},
|
||||
{"Бердск, Ленина 5", "", false},
|
||||
{"Томск", "", false},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
got, ok := dict.MatchCity(tt.query)
|
||||
if ok != tt.ok || got.Name != tt.want {
|
||||
t.Errorf("MatchCity(%q) = %+v, %v; want %q, %v", tt.query, got, ok, tt.want, tt.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzParse(f *testing.F) {
|
||||
for _, seed := range []string{
|
||||
"",
|
||||
"2-я Станционная улица 1/1 стр.",
|
||||
"нск, Ленина 5",
|
||||
"Бердск, Ленина 5 к2",
|
||||
"г. Новосибирск, ул. Кошурникова, д. 11/1 кв. 5",
|
||||
"\x00\xff",
|
||||
} {
|
||||
f.Add(seed)
|
||||
}
|
||||
parser := NewParser(testDictionary(), "")
|
||||
f.Fuzz(func(t *testing.T, raw string) {
|
||||
first := parser.Parse(raw)
|
||||
second := parser.Parse(raw)
|
||||
if first.StreetKey != second.StreetKey {
|
||||
t.Fatalf("non-deterministic street key for %q: %q vs %q", raw, first.StreetKey, second.StreetKey)
|
||||
}
|
||||
if NormalizeText(raw) != NormalizeText(NormalizeText(raw)) {
|
||||
t.Fatalf("NormalizeText is not idempotent for %q", raw)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package address
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// City is one entry of the configurable city dictionary.
|
||||
//
|
||||
// Aliases are additional surface forms that refer to the city: common
|
||||
// abbreviations ("нск"), the region name ("новосибирская область") and so on.
|
||||
// The canonical Name is always accepted as an alias too. Matching is
|
||||
// token-based, case-insensitive and "ё"-insensitive, so aliases should be
|
||||
// written naturally.
|
||||
type City struct {
|
||||
Name string `json:"name"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
}
|
||||
|
||||
// Dictionary is the immutable, deployment-specific vocabulary of the parser:
|
||||
// cities, filler words, street types and ordinals.
|
||||
//
|
||||
// Keeping this data in a value instead of package-level maps makes the parser
|
||||
// reusable across deployments: a service in Novosibirsk and a service in Tomsk
|
||||
// can run the same code with different city lists, without recompilation.
|
||||
type Dictionary struct {
|
||||
cities []cityAlias
|
||||
|
||||
filler map[string]struct{}
|
||||
streetTypes map[string]string
|
||||
ordinalWords map[string]string
|
||||
ordinalSuffixes map[string]struct{}
|
||||
}
|
||||
|
||||
// cityAlias is one token sequence that resolves to a city.
|
||||
type cityAlias struct {
|
||||
tokens []string
|
||||
name string
|
||||
key string
|
||||
}
|
||||
|
||||
// NewDictionary builds a dictionary with the built-in federal vocabulary and
|
||||
// the supplied cities. The input slice is copied; mutating it later does not
|
||||
// affect the dictionary.
|
||||
func NewDictionary(cities []City) *Dictionary {
|
||||
d := &Dictionary{
|
||||
filler: cloneSet(defaultFillerWords),
|
||||
streetTypes: cloneMap(defaultStreetTypes),
|
||||
ordinalWords: cloneMap(defaultOrdinalWords),
|
||||
ordinalSuffixes: cloneSet(defaultOrdinalSuffixes),
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
for _, c := range cities {
|
||||
name := strings.TrimSpace(c.Name)
|
||||
cityKey := NormalizeText(name)
|
||||
if name == "" || cityKey == "" {
|
||||
continue
|
||||
}
|
||||
aliases := append([]string{name}, c.Aliases...)
|
||||
for _, alias := range aliases {
|
||||
tokens := strings.Fields(NormalizeText(alias))
|
||||
if len(tokens) == 0 {
|
||||
continue
|
||||
}
|
||||
dedup := cityKey + "|" + strings.Join(tokens, " ")
|
||||
if _, ok := seen[dedup]; ok {
|
||||
continue
|
||||
}
|
||||
seen[dedup] = struct{}{}
|
||||
d.cities = append(d.cities, cityAlias{
|
||||
tokens: tokens,
|
||||
name: name,
|
||||
key: cityKey,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Longest aliases win, so "новосибирская область" is preferred over a
|
||||
// hypothetical shorter "новосибирская". The key is the tie-breaker that
|
||||
// keeps the order deterministic when two cities share an alias.
|
||||
d.sortCities()
|
||||
return d
|
||||
}
|
||||
|
||||
// sortCities orders aliases by specificity: more tokens first, then longer
|
||||
// text, then key. The order decides which city wins when aliases overlap.
|
||||
func (d *Dictionary) sortCities() {
|
||||
sort.SliceStable(d.cities, func(i, j int) bool {
|
||||
a, b := d.cities[i], d.cities[j]
|
||||
if len(a.tokens) != len(b.tokens) {
|
||||
return len(a.tokens) > len(b.tokens)
|
||||
}
|
||||
as, bs := strings.Join(a.tokens, " "), strings.Join(b.tokens, " ")
|
||||
if len(as) != len(bs) {
|
||||
return len(as) > len(bs)
|
||||
}
|
||||
if a.key != b.key {
|
||||
return a.key < b.key
|
||||
}
|
||||
return as < bs
|
||||
})
|
||||
}
|
||||
|
||||
// withDefaultCity returns a dictionary that also recognises name as a city.
|
||||
// It lets a single-city deployment use "-city Новосибирск" (or default_city)
|
||||
// without duplicating the canonical name in the aliases list.
|
||||
func (d *Dictionary) withDefaultCity(name string) *Dictionary {
|
||||
key := NormalizeText(name)
|
||||
if key == "" {
|
||||
return d
|
||||
}
|
||||
for _, a := range d.cities {
|
||||
if a.key == key {
|
||||
return d
|
||||
}
|
||||
}
|
||||
out := *d
|
||||
out.cities = append(append([]cityAlias(nil), d.cities...), cityAlias{
|
||||
tokens: strings.Fields(key),
|
||||
name: name,
|
||||
key: key,
|
||||
})
|
||||
out.sortCities()
|
||||
return &out
|
||||
}
|
||||
|
||||
// DefaultDictionary returns the built-in vocabulary without cities. It is the
|
||||
// parser used by address.Parse for callers that do not need city awareness.
|
||||
func DefaultDictionary() *Dictionary {
|
||||
return NewDictionary(nil)
|
||||
}
|
||||
|
||||
// MatchCity reports whether query consists only of a city name/alias (plus
|
||||
// generic filler such as "г.") and returns the canonical city. It is the
|
||||
// city-only counterpart of Parse, useful for projects that normalize city
|
||||
// names without a street.
|
||||
func (d *Dictionary) MatchCity(query string) (City, bool) {
|
||||
name, _, rest, ok := d.matchCity(strings.Fields(NormalizeText(query)))
|
||||
if !ok {
|
||||
return City{}, false
|
||||
}
|
||||
for _, tok := range rest {
|
||||
if !d.isFiller(tok) {
|
||||
return City{}, false
|
||||
}
|
||||
}
|
||||
return City{Name: name}, true
|
||||
}
|
||||
|
||||
// matchCity finds the longest city alias in tokens and returns the city plus
|
||||
// the tokens with the alias removed. Matching is contiguous on token
|
||||
// boundaries, so "нск" never matches inside another word.
|
||||
func (d *Dictionary) matchCity(tokens []string) (name, key string, rest []string, ok bool) {
|
||||
for _, alias := range d.cities {
|
||||
if idx := indexTokens(tokens, alias.tokens); idx >= 0 {
|
||||
rest = make([]string, 0, len(tokens)-len(alias.tokens))
|
||||
rest = append(rest, tokens[:idx]...)
|
||||
rest = append(rest, tokens[idx+len(alias.tokens):]...)
|
||||
return alias.name, alias.key, rest, true
|
||||
}
|
||||
}
|
||||
return "", "", tokens, false
|
||||
}
|
||||
|
||||
// indexTokens returns the start of the first contiguous occurrence of want in
|
||||
// tokens, or -1.
|
||||
func indexTokens(tokens, want []string) int {
|
||||
if len(want) == 0 || len(want) > len(tokens) {
|
||||
return -1
|
||||
}
|
||||
for i := 0; i+len(want) <= len(tokens); i++ {
|
||||
matched := true
|
||||
for j := range want {
|
||||
if tokens[i+j] != want[j] {
|
||||
matched = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (d *Dictionary) isFiller(tok string) bool {
|
||||
_, ok := d.filler[tok]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (d *Dictionary) streetType(tok string) (string, bool) {
|
||||
t, ok := d.streetTypes[tok]
|
||||
return t, ok
|
||||
}
|
||||
|
||||
func (d *Dictionary) ordinalWord(tok string) (string, bool) {
|
||||
o, ok := d.ordinalWords[tok]
|
||||
return o, ok
|
||||
}
|
||||
|
||||
func (d *Dictionary) isOrdinalSuffix(tok string) bool {
|
||||
_, ok := d.ordinalSuffixes[tok]
|
||||
return ok
|
||||
}
|
||||
|
||||
func cloneSet(src map[string]bool) map[string]struct{} {
|
||||
dst := make(map[string]struct{}, len(src))
|
||||
for k, v := range src {
|
||||
if v {
|
||||
dst[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func cloneMap(src map[string]string) map[string]string {
|
||||
dst := make(map[string]string, len(src))
|
||||
for k, v := range src {
|
||||
dst[k] = v
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// defaultFillerWords are tokens that carry no street-name meaning and that
|
||||
// people routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
|
||||
//
|
||||
// City names are deliberately NOT listed here: they come from Dictionary, so
|
||||
// the same code can serve any city (or many cities) without edits.
|
||||
var defaultFillerWords = map[string]bool{
|
||||
"город": true, "г": true,
|
||||
"область": true, "обл": true,
|
||||
"район": true, "дом": true, "д": true,
|
||||
}
|
||||
|
||||
// defaultStreetTypes maps a surface form (including common abbreviations) to a
|
||||
// single canonical street type. The type is informational for matching (we
|
||||
// strip it from the key) but useful for disambiguation and display.
|
||||
var defaultStreetTypes = map[string]string{
|
||||
"ул": "улица", "улица": "улица",
|
||||
"пер": "переулок", "пер-к": "переулок", "переулок": "переулок",
|
||||
"пр": "проезд", "проезд": "проезд",
|
||||
"пр-т": "проспект", "просп": "проспект", "проспект": "проспект",
|
||||
"ш": "шоссе", "шоссе": "шоссе",
|
||||
"мкр": "микрорайон", "микрорайон": "микрорайон",
|
||||
"туп": "тупик", "тупик": "тупик",
|
||||
"спуск": "спуск",
|
||||
"б-р": "бульвар", "бул": "бульвар", "бульвар": "бульвар",
|
||||
"пл": "площадь", "площадь": "площадь",
|
||||
"пос": "поселок", "поселок": "поселок",
|
||||
"тракт": "тракт",
|
||||
"аллея": "аллея",
|
||||
"наб": "набережная", "набережная": "набережная",
|
||||
"кольцо": "кольцо",
|
||||
"просека": "просека",
|
||||
"квартал": "квартал",
|
||||
"территория": "территория",
|
||||
"дорога": "дорога",
|
||||
"разъезд": "разъезд",
|
||||
}
|
||||
|
||||
// defaultOrdinalWords maps spelled-out ordinals to their digit form. Only the
|
||||
// forms that actually occur in practice are listed; unknown words are treated
|
||||
// as part of the street name.
|
||||
var defaultOrdinalWords = map[string]string{
|
||||
"первый": "1", "первая": "1", "первое": "1",
|
||||
"второй": "2", "вторая": "2", "второе": "2",
|
||||
"третий": "3", "третья": "3", "третье": "3",
|
||||
"четвертый": "4", "четвертая": "4", "четвертое": "4",
|
||||
"пятый": "5", "пятая": "5", "пятое": "5",
|
||||
"шестой": "6", "шестая": "6", "шестое": "6",
|
||||
"седьмой": "7", "седьмая": "7", "седьмое": "7",
|
||||
"восьмой": "8", "восьмая": "8", "восьмое": "8",
|
||||
"девятый": "9", "девятая": "9", "девятое": "9",
|
||||
"десятый": "10", "десятая": "10", "десятое": "10",
|
||||
}
|
||||
|
||||
// defaultOrdinalSuffixes are the gender/number endings that may follow an
|
||||
// ordinal digit when it is written as a separate token ("2 я", "10 й").
|
||||
var defaultOrdinalSuffixes = map[string]bool{
|
||||
"я": true, "й": true, "е": true,
|
||||
"ая": true, "ый": true, "ой": true, "ое": true,
|
||||
"ий": true, "ья": true, "ье": true, "ьи": true,
|
||||
}
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
var fold = cases.Fold()
|
||||
|
||||
// reTrailingParenthetical matches a trailing "(...)" group, e.g. the district
|
||||
// suffix that the outages portal appends: "Медкадры улица, 1 (Заельцовский
|
||||
// suffix that source systems append: "Медкадры улица, 1 (Заельцовский
|
||||
// район)". Such administrative noise must not be treated as part of the house.
|
||||
var reTrailingParenthetical = regexp.MustCompile(`\s*\([^)]*\)\s*$`)
|
||||
|
||||
|
||||
@@ -24,42 +24,121 @@ var (
|
||||
// Trailing apartment part: " кв 5", " квартира 12", " кв. 7а". Removed
|
||||
// before house parsing so it cannot leak into the street name.
|
||||
reApartment = regexp.MustCompile(`(?:^|\s)(?:квартира|кв)\.?(?:\s+\d+[а-я]?)?\s*$`)
|
||||
|
||||
// Spoken fractions: "18 дробь 1" is the same house unit as "18/1".
|
||||
reFractionWord = regexp.MustCompile(`\s+(?:дробь|дробью)\s+`)
|
||||
)
|
||||
|
||||
// Parse decomposes a free-form address into its structured components. It is
|
||||
// total: unparseable input still yields a non-nil ParsedAddress whose street
|
||||
// part is best-effort and whose HasHouse is false.
|
||||
// Parser turns free-form address strings into structured addresses using a
|
||||
// deployment-specific Dictionary. It is immutable and safe for concurrent use.
|
||||
type Parser struct {
|
||||
dict *Dictionary
|
||||
defaultCityName string
|
||||
defaultCityKey string
|
||||
}
|
||||
|
||||
// NewParser builds a parser. defaultCity is used when the query does not name a
|
||||
// city and the dictionary has no match; pass an empty string for deployments
|
||||
// where the city must be present in every query.
|
||||
func NewParser(dict *Dictionary, defaultCity string) *Parser {
|
||||
if dict == nil {
|
||||
dict = DefaultDictionary()
|
||||
}
|
||||
p := &Parser{}
|
||||
if name := strings.TrimSpace(defaultCity); name != "" {
|
||||
dict = dict.withDefaultCity(name)
|
||||
p.defaultCityName = name
|
||||
p.defaultCityKey = NormalizeText(name)
|
||||
}
|
||||
p.dict = dict
|
||||
return p
|
||||
}
|
||||
|
||||
// defaultParser is used by the package-level Parse convenience function. It has
|
||||
// no cities and is therefore fully backward compatible: callers that need city
|
||||
// awareness construct their own Parser.
|
||||
var defaultParser = NewParser(DefaultDictionary(), "")
|
||||
|
||||
// DefaultParser returns the shared city-less parser.
|
||||
func DefaultParser() *Parser {
|
||||
return defaultParser
|
||||
}
|
||||
|
||||
// Parse decomposes a free-form address with the default (city-less) parser.
|
||||
func Parse(raw string) ParsedAddress {
|
||||
return defaultParser.Parse(raw)
|
||||
}
|
||||
|
||||
// Parse decomposes a free-form address into its structured components. It is
|
||||
// total: unparseable input still yields a usable ParsedAddress whose street
|
||||
// part is best-effort and whose HasHouse is false.
|
||||
func (p *Parser) Parse(raw string) ParsedAddress {
|
||||
return p.parse(raw, "")
|
||||
}
|
||||
|
||||
// ParseInCity parses raw and forces cityName as the result's city. It is used
|
||||
// by the seed pipeline, where the city is an explicit field of the source row
|
||||
// and must win over whatever the address text contains.
|
||||
func (p *Parser) ParseInCity(raw, cityName string) ParsedAddress {
|
||||
return p.parse(raw, cityName)
|
||||
}
|
||||
|
||||
// MatchCity reports whether query is a city name/alias and returns its
|
||||
// canonical form. It does not guess a city from an arbitrary address.
|
||||
func (p *Parser) MatchCity(query string) (City, bool) {
|
||||
return p.dict.MatchCity(query)
|
||||
}
|
||||
|
||||
func (p *Parser) parse(raw, forcedCity string) ParsedAddress {
|
||||
normalized := NormalizeText(raw)
|
||||
// A trailing apartment ("кв 5") is not part of the address identity.
|
||||
normalized = strings.TrimSpace(reApartment.ReplaceAllString(normalized, ""))
|
||||
p := ParsedAddress{Raw: raw, Normalized: normalized}
|
||||
// Collapse the spoken fraction marker to the canonical separator before
|
||||
// the right-to-left house parser runs: "18 дробь 1" -> "18/1".
|
||||
normalized = reFractionWord.ReplaceAllString(normalized, "/")
|
||||
parsed := ParsedAddress{Raw: raw, Normalized: normalized}
|
||||
|
||||
rest := normalized
|
||||
if cityName, cityKey, restTokens, ok := p.dict.matchCity(strings.Fields(rest)); ok {
|
||||
parsed.City = cityName
|
||||
parsed.CityKey = cityKey
|
||||
rest = strings.Join(restTokens, " ")
|
||||
}
|
||||
if forcedCity = strings.TrimSpace(forcedCity); forcedCity != "" {
|
||||
parsed.City = forcedCity
|
||||
parsed.CityKey = NormalizeText(forcedCity)
|
||||
} else if parsed.City == "" && p.defaultCityName != "" {
|
||||
// A single-city deployment can omit the city in queries. In a
|
||||
// multi-city deployment default_city should stay empty: guessing a
|
||||
// city is exactly the kind of silent error this service avoids.
|
||||
parsed.City = p.defaultCityName
|
||||
parsed.CityKey = p.defaultCityKey
|
||||
}
|
||||
|
||||
if g, start, ok := matchSuffix(reStructure, rest); ok {
|
||||
p.Structure = trimLeadingZeros(g[1])
|
||||
parsed.Structure = trimLeadingZeros(g[1])
|
||||
rest = strings.TrimSpace(rest[:start])
|
||||
}
|
||||
if g, start, ok := matchSuffix(reBuilding, rest); ok {
|
||||
// The regex also matches a bare "к"; only accept it when a digit or a
|
||||
// letter actually follows, otherwise it is not a corpus marker.
|
||||
if g[1] != "" || g[2] != "" {
|
||||
p.Building = trimLeadingZeros(g[1])
|
||||
p.BuildingLetter = firstNonEmpty(g[2], g[4])
|
||||
p.BuildingFrac = trimLeadingZeros(g[3])
|
||||
parsed.Building = trimLeadingZeros(g[1])
|
||||
parsed.BuildingLetter = firstNonEmpty(g[2], g[4])
|
||||
parsed.BuildingFrac = trimLeadingZeros(g[3])
|
||||
rest = strings.TrimSpace(rest[:start])
|
||||
}
|
||||
}
|
||||
if g, start, ok := matchSuffix(reHouse, rest); ok {
|
||||
p.House = trimLeadingZeros(g[1])
|
||||
p.HouseLetter = firstNonEmpty(g[2], g[4])
|
||||
p.HouseFrac = trimLeadingZeros(g[3])
|
||||
p.HasHouse = true
|
||||
parsed.House = trimLeadingZeros(g[1])
|
||||
parsed.HouseLetter = firstNonEmpty(g[2], g[4])
|
||||
parsed.HouseFrac = trimLeadingZeros(g[3])
|
||||
parsed.HasHouse = true
|
||||
rest = strings.TrimSpace(rest[:start])
|
||||
}
|
||||
|
||||
parseStreet(&p, rest)
|
||||
return p
|
||||
p.parseStreet(&parsed, rest)
|
||||
return parsed
|
||||
}
|
||||
|
||||
// firstNonEmpty returns the first non-empty argument. House/building letters
|
||||
@@ -73,8 +152,8 @@ func firstNonEmpty(values ...string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseStreet fills the street-related fields of p from the leftover string
|
||||
// (the address with its house block removed).
|
||||
// parseStreet fills the street-related fields of out from the leftover string
|
||||
// (the address with its city and house block removed).
|
||||
//
|
||||
// All recognised type tokens are removed from the name. This matters for
|
||||
// registry rows such as "Гусинобродский Тракт улица 21", where both "Тракт"
|
||||
@@ -82,7 +161,7 @@ func firstNonEmpty(values ...string) string {
|
||||
// says "Гусинобродский 21". The first type token becomes the canonical type.
|
||||
// If removing them would leave the name empty (a street literally called
|
||||
// "Набережная"), the type token is kept as the name.
|
||||
func parseStreet(p *ParsedAddress, s string) {
|
||||
func (p *Parser) parseStreet(out *ParsedAddress, s string) {
|
||||
tokens := strings.Fields(s)
|
||||
name := make([]string, 0, len(tokens))
|
||||
streetType := ""
|
||||
@@ -90,10 +169,10 @@ func parseStreet(p *ParsedAddress, s string) {
|
||||
for _, tok := range tokens {
|
||||
// Filler tokens ("г", "город", "дом", "область", ...) never belong to
|
||||
// the street name.
|
||||
if fillerWords[tok] {
|
||||
if p.dict.isFiller(tok) {
|
||||
continue
|
||||
}
|
||||
if t, ok := streetTypes[tok]; ok {
|
||||
if t, ok := p.dict.streetType(tok); ok {
|
||||
if streetType == "" {
|
||||
streetType = t
|
||||
}
|
||||
@@ -102,55 +181,55 @@ func parseStreet(p *ParsedAddress, s string) {
|
||||
name = append(name, tok)
|
||||
}
|
||||
|
||||
p.StreetType = streetType
|
||||
out.StreetType = streetType
|
||||
if len(name) == 0 && streetType != "" {
|
||||
name = []string{streetType}
|
||||
}
|
||||
if len(name) == 0 {
|
||||
p.StreetKey = streetKey("", "")
|
||||
out.StreetKey = streetKey(out.CityKey, "", "")
|
||||
return
|
||||
}
|
||||
|
||||
name = extractOrdinal(p, name)
|
||||
p.NameDisplay = strings.Join(name, " ")
|
||||
name = p.extractOrdinal(out, name)
|
||||
out.NameDisplay = strings.Join(name, " ")
|
||||
|
||||
sorted := append([]string(nil), name...)
|
||||
sort.Strings(sorted)
|
||||
p.NameKey = strings.Join(sorted, " ")
|
||||
p.StreetKey = streetKey(p.Ordinal, p.NameKey)
|
||||
out.NameKey = strings.Join(sorted, " ")
|
||||
out.StreetKey = streetKey(out.CityKey, out.Ordinal, out.NameKey)
|
||||
}
|
||||
|
||||
// extractOrdinal recognises an ordinal and removes it from the name tokens.
|
||||
// It handles both the usual leading form ("2-я Сухарная") and an ordinal that
|
||||
// appears after the name ("Сухарная 2-я улица"), which both people and models
|
||||
// produce.
|
||||
func extractOrdinal(p *ParsedAddress, tokens []string) []string {
|
||||
func (p *Parser) extractOrdinal(out *ParsedAddress, tokens []string) []string {
|
||||
if len(tokens) == 0 {
|
||||
return tokens
|
||||
}
|
||||
|
||||
// Leading forms first: they are by far the most common.
|
||||
if m := reOrdinalToken.FindStringSubmatch(tokens[0]); m != nil {
|
||||
p.Ordinal = normalizeOrdinal(m[1])
|
||||
out.Ordinal = normalizeOrdinal(m[1])
|
||||
return tokens[1:]
|
||||
}
|
||||
if isDigits(tokens[0]) && len(tokens) >= 2 && ordinalSuffixes[tokens[1]] {
|
||||
p.Ordinal = normalizeOrdinal(tokens[0])
|
||||
if isDigits(tokens[0]) && len(tokens) >= 2 && p.dict.isOrdinalSuffix(tokens[1]) {
|
||||
out.Ordinal = normalizeOrdinal(tokens[0])
|
||||
return tokens[2:]
|
||||
}
|
||||
if d, ok := ordinalWords[tokens[0]]; ok {
|
||||
p.Ordinal = normalizeOrdinal(d)
|
||||
if d, ok := p.dict.ordinalWord(tokens[0]); ok {
|
||||
out.Ordinal = normalizeOrdinal(d)
|
||||
return tokens[1:]
|
||||
}
|
||||
|
||||
// Otherwise look for an ordinal anywhere among the name tokens.
|
||||
for i, tok := range tokens {
|
||||
if m := reOrdinalToken.FindStringSubmatch(tok); m != nil {
|
||||
p.Ordinal = normalizeOrdinal(m[1])
|
||||
out.Ordinal = normalizeOrdinal(m[1])
|
||||
return removeRange(tokens, i, i+1)
|
||||
}
|
||||
if isDigits(tok) && i+1 < len(tokens) && ordinalSuffixes[tokens[i+1]] {
|
||||
p.Ordinal = normalizeOrdinal(tok)
|
||||
if isDigits(tok) && i+1 < len(tokens) && p.dict.isOrdinalSuffix(tokens[i+1]) {
|
||||
out.Ordinal = normalizeOrdinal(tok)
|
||||
return removeRange(tokens, i, i+2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,6 +71,12 @@ func TestParseHouseBlock(t *testing.T) {
|
||||
wantName: "выборная", wantType: "улица",
|
||||
wantHouse: "45", wantLetter: "а", wantFrac: "4", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "spoken fraction word",
|
||||
raw: "Горская улица 18 дробь 1",
|
||||
wantName: "горская", wantType: "улица",
|
||||
wantHouse: "18", wantFrac: "1", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "corpus letter without digit",
|
||||
raw: "1-я Ельцовка улица 1 кД",
|
||||
@@ -96,7 +102,7 @@ func TestParseHouseBlock(t *testing.T) {
|
||||
wantHouse: "11", wantFrac: "1", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "portal district suffix in parentheses",
|
||||
name: "district suffix in parentheses",
|
||||
raw: "Медкадры улица, 1 (Заельцовский район)",
|
||||
wantName: "медкадры", wantType: "улица",
|
||||
wantHouse: "1", wantHas: true,
|
||||
|
||||
@@ -2,73 +2,19 @@ package address
|
||||
|
||||
import "strings"
|
||||
|
||||
// fillerWords are tokens that carry no street-name meaning and that people
|
||||
// routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
|
||||
// Dropping them makes the service tolerant of natural phrasing. District
|
||||
// adjectives are deliberately NOT dropped: they can legitimately be part of a
|
||||
// street name.
|
||||
var fillerWords = map[string]bool{
|
||||
"новосибирск": true, "нск": true, "нсо": true,
|
||||
"город": true, "г": true,
|
||||
"область": true, "обл": true,
|
||||
"район": true, "дом": true, "д": true,
|
||||
}
|
||||
|
||||
// streetTypes maps a surface form (including common abbreviations) to a single
|
||||
// canonical street type. The type is informational for matching (we strip it
|
||||
// from the key) but useful for disambiguation and display.
|
||||
var streetTypes = map[string]string{
|
||||
"ул": "улица", "улица": "улица",
|
||||
"пер": "переулок", "пер-к": "переулок", "переулок": "переулок",
|
||||
"пр": "проезд", "проезд": "проезд",
|
||||
"пр-т": "проспект", "просп": "проспект", "проспект": "проспект",
|
||||
"ш": "шоссе", "шоссе": "шоссе",
|
||||
"мкр": "микрорайон", "микрорайон": "микрорайон",
|
||||
"туп": "тупик", "тупик": "тупик",
|
||||
"спуск": "спуск",
|
||||
"б-р": "бульвар", "бул": "бульвар", "бульвар": "бульвар",
|
||||
"пл": "площадь", "площадь": "площадь",
|
||||
"пос": "поселок", "поселок": "поселок",
|
||||
"тракт": "тракт",
|
||||
"аллея": "аллея",
|
||||
"наб": "набережная", "набережная": "набережная",
|
||||
"кольцо": "кольцо",
|
||||
"просека": "просека",
|
||||
"квартал": "квартал",
|
||||
"территория": "территория",
|
||||
"дорога": "дорога",
|
||||
"разъезд": "разъезд",
|
||||
}
|
||||
|
||||
// ordinalWords maps spelled-out ordinals to their digit form. Only the forms
|
||||
// that actually occur in practice are listed; unknown words are treated as
|
||||
// part of the street name.
|
||||
var ordinalWords = map[string]string{"первый": "1", "первая": "1", "первое": "1",
|
||||
"второй": "2", "вторая": "2", "второе": "2",
|
||||
"третий": "3", "третья": "3", "третье": "3",
|
||||
"четвертый": "4", "четвертая": "4", "четвертое": "4",
|
||||
"пятый": "5", "пятая": "5", "пятое": "5",
|
||||
"шестой": "6", "шестая": "6", "шестое": "6",
|
||||
"седьмой": "7", "седьмая": "7", "седьмое": "7",
|
||||
"восьмой": "8", "восьмая": "8", "восьмое": "8",
|
||||
"девятый": "9", "девятая": "9", "девятое": "9",
|
||||
"десятый": "10", "десятая": "10", "десятое": "10",
|
||||
}
|
||||
|
||||
// ordinalSuffixes are the gender/number endings that may follow an ordinal
|
||||
// digit when it is written as a separate token ("2 я", "10 й").
|
||||
var ordinalSuffixes = map[string]bool{
|
||||
"я": true, "й": true, "е": true,
|
||||
"ая": true, "ый": true, "ой": true, "ое": true,
|
||||
"ий": true, "ья": true, "ье": true, "ьи": true,
|
||||
}
|
||||
|
||||
// streetKey builds the canonical, type-independent street key:
|
||||
// "<ordinal>|<sorted name tokens>". The type word is dropped so that a user
|
||||
// who omits it ("станционная 1" instead of "Станционная улица 1") still
|
||||
// matches; type confusion is surfaced later as ambiguity.
|
||||
func streetKey(ordinal, nameKey string) string {
|
||||
return ordinal + "|" + nameKey
|
||||
// streetKey builds the canonical, city-aware, type-independent street key:
|
||||
//
|
||||
// "<cityKey>|<ordinal>|<sorted name tokens>"
|
||||
//
|
||||
// The city is part of the key so that "Ленина" in Novosibirsk and "Ленина" in
|
||||
// Berdsk are different streets. The type word is dropped so that a user who
|
||||
// omits it ("станционная 1" instead of "Станционная улица 1") still matches;
|
||||
// type confusion is surfaced later as ambiguity.
|
||||
//
|
||||
// None of the components may contain "|": NormalizeText folds all punctuation
|
||||
// that could appear in a name into spaces, and ordinals are digits only.
|
||||
func streetKey(cityKey, ordinal, nameKey string) string {
|
||||
return cityKey + "|" + ordinal + "|" + nameKey
|
||||
}
|
||||
|
||||
// normalizeOrdinal trims leading zeros so "02-я" and "2-я" agree.
|
||||
|
||||
@@ -10,7 +10,7 @@ package address
|
||||
// Status is the machine-readable outcome of an address resolution.
|
||||
//
|
||||
// It is intentionally explicit: callers must never have to guess whether an
|
||||
// empty result means "not found" or "the upstream failed".
|
||||
// empty result means "not found" or "the resolver refused to choose".
|
||||
type Status string
|
||||
|
||||
const (
|
||||
@@ -25,11 +25,6 @@ const (
|
||||
StatusAmbiguous Status = "ambiguous"
|
||||
// StatusNotFound means nothing in the registry matched.
|
||||
StatusNotFound Status = "not_found"
|
||||
// StatusPortalTimeout means the external outages portal did not answer in
|
||||
// time. It is an infrastructure outcome, not "no outages".
|
||||
StatusPortalTimeout Status = "portal_timeout"
|
||||
// StatusInfra is a generic terminal infrastructure failure.
|
||||
StatusInfra Status = "infra_error"
|
||||
)
|
||||
|
||||
// ParsedAddress is the structured form of a free-form address string.
|
||||
@@ -41,6 +36,13 @@ type ParsedAddress struct {
|
||||
Raw string `json:"raw"`
|
||||
Normalized string `json:"normalized"`
|
||||
|
||||
// City is the canonical city name when the query names one (or when the
|
||||
// parser is configured with a default city). CityKey is its normalized
|
||||
// form. The city is part of StreetKey, so identical street names in
|
||||
// different cities never collide.
|
||||
City string `json:"city,omitempty"`
|
||||
CityKey string `json:"city_key,omitempty"`
|
||||
|
||||
// Street components.
|
||||
//
|
||||
// NameKey is the order-insensitive canonical name (tokens sorted), so
|
||||
@@ -73,6 +75,9 @@ type Entry struct {
|
||||
Address string `json:"address"`
|
||||
District string `json:"district,omitempty"`
|
||||
|
||||
City string `json:"city,omitempty"`
|
||||
CityKey string `json:"city_key,omitempty"`
|
||||
|
||||
Ordinal string `json:"ordinal,omitempty"`
|
||||
NameKey string `json:"name_key,omitempty"`
|
||||
NameDisplay string `json:"name_display,omitempty"`
|
||||
|
||||
Reference in New Issue
Block a user