refactor!: make addrkit a multi-city address normalizer

- remove the events/outages module, ArcGIS adapter, portal config and event statuses
- add a configurable city dictionary/parser and make City part of StreetKey
- seed: read city/city_name, namespace IDs by city, keep structures distinct
- fix reliability: schema v2 with fail-closed load, deterministic ordering,
  input limits, config validation, atomic DB publish, LLM response caps
- add read-only shadow mode comparing the old ES path with addrkit
- breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
Maksim Totmin
2026-10-27 14:35:51 +07:00
parent a0b4cb2b14
commit 17ff5c591f
38 changed files with 2978 additions and 1164 deletions
+141
View File
@@ -0,0 +1,141 @@
package address
import "testing"
func testDictionary() *Dictionary {
return NewDictionary([]City{
{Name: "Новосибирск", Aliases: []string{"нск", "новосиб", "новосибирская область", "нсо"}},
{Name: "Бердск"},
})
}
func TestParseCityFromAliases(t *testing.T) {
parser := NewParser(testDictionary(), "")
tests := []struct {
raw string
city string
nameKey string
}{
{"нск, 2-я Станционная улица 1", "Новосибирск", "станционная"},
{"НСК Ленина 5", "Новосибирск", "ленина"},
{"Новосибирская область, Станционная 1", "Новосибирск", "станционная"},
{"Бердск, Ленина 5", "Бердск", "ленина"},
{"г. Новосибирск, ул. Ленина 5", "Новосибирск", "ленина"},
}
for _, tt := range tests {
got := parser.Parse(tt.raw)
if got.City != tt.city {
t.Errorf("%q: City = %q, want %q", tt.raw, got.City, tt.city)
}
if got.NameKey != tt.nameKey {
t.Errorf("%q: NameKey = %q, want %q", tt.raw, got.NameKey, tt.nameKey)
}
}
}
func TestParseCityIsPartOfStreetKey(t *testing.T) {
parser := NewParser(testDictionary(), "")
nsk := parser.Parse("Новосибирск, Ленина 5")
berdsk := parser.Parse("Бердск, Ленина 5")
if nsk.StreetKey == berdsk.StreetKey {
t.Fatalf("same street key for different cities: %q", nsk.StreetKey)
}
if nsk.CityKey != "новосибирск" || berdsk.CityKey != "бердск" {
t.Fatalf("city keys = %q / %q", nsk.CityKey, berdsk.CityKey)
}
}
func TestParseDefaultCity(t *testing.T) {
parser := NewParser(testDictionary(), "Новосибирск")
got := parser.Parse("Ленина 5")
if got.City != "Новосибирск" || got.CityKey != "новосибирск" {
t.Fatalf("City = %q / %q", got.City, got.CityKey)
}
// The default city is also recognised when the user spells it out, even
// if the dictionary contains only aliases.
explicit := NewParser(DefaultDictionary(), "Новосибирск").Parse("г. Новосибирск, Ленина 5")
if explicit.City != "Новосибирск" || explicit.NameKey != "ленина" {
t.Fatalf("explicit default city: %+v", explicit)
}
}
func TestParseInCityOverridesDetectedCity(t *testing.T) {
parser := NewParser(testDictionary(), "")
got := parser.ParseInCity("Бердск, Ленина 5", "Новосибирск")
if got.City != "Новосибирск" {
t.Fatalf("City = %q, want forced Новосибирск", got.City)
}
if got.NameKey != "ленина" {
t.Fatalf("NameKey = %q", got.NameKey)
}
}
func TestParseWithoutCityKeepsCityKeyEmpty(t *testing.T) {
parser := NewParser(testDictionary(), "")
got := parser.Parse("Ленина 5")
if got.City != "" || got.CityKey != "" {
t.Fatalf("City = %q / %q, want empty", got.City, got.CityKey)
}
}
func TestNormalizeTextStillLeavesUnknownCityTokens(t *testing.T) {
// A city that is not in the dictionary must not be silently dropped: the
// resolver should fail rather than guess another city.
parser := NewParser(testDictionary(), "")
got := parser.Parse("Томск, Ленина 5")
if got.City != "" {
t.Fatalf("City = %q, want empty for unknown city", got.City)
}
if got.NameKey == "ленина" {
t.Fatalf("unknown city token disappeared from the street name")
}
}
func TestMatchCity(t *testing.T) {
dict := testDictionary()
cases := []struct {
query string
want string
ok bool
}{
{"нск", "Новосибирск", true},
{"г. Новосибирск", "Новосибирск", true},
{"Новосибирская область", "Новосибирск", true},
{"Бердск", "Бердск", true},
{"Бердск, Ленина 5", "", false},
{"Томск", "", false},
}
for _, tt := range cases {
got, ok := dict.MatchCity(tt.query)
if ok != tt.ok || got.Name != tt.want {
t.Errorf("MatchCity(%q) = %+v, %v; want %q, %v", tt.query, got, ok, tt.want, tt.ok)
}
}
}
func FuzzParse(f *testing.F) {
for _, seed := range []string{
"",
"2-я Станционная улица 1/1 стр.",
"нск, Ленина 5",
"Бердск, Ленина 5 к2",
"г. Новосибирск, ул. Кошурникова, д. 11/1 кв. 5",
"\x00\xff",
} {
f.Add(seed)
}
parser := NewParser(testDictionary(), "")
f.Fuzz(func(t *testing.T, raw string) {
first := parser.Parse(raw)
second := parser.Parse(raw)
if first.StreetKey != second.StreetKey {
t.Fatalf("non-deterministic street key for %q: %q vs %q", raw, first.StreetKey, second.StreetKey)
}
if NormalizeText(raw) != NormalizeText(NormalizeText(raw)) {
t.Fatalf("NormalizeText is not idempotent for %q", raw)
}
})
}
+284
View File
@@ -0,0 +1,284 @@
package address
import (
"sort"
"strings"
)
// City is one entry of the configurable city dictionary.
//
// Aliases are additional surface forms that refer to the city: common
// abbreviations ("нск"), the region name ("новосибирская область") and so on.
// The canonical Name is always accepted as an alias too. Matching is
// token-based, case-insensitive and "ё"-insensitive, so aliases should be
// written naturally.
type City struct {
Name string `json:"name"`
Aliases []string `json:"aliases,omitempty"`
}
// Dictionary is the immutable, deployment-specific vocabulary of the parser:
// cities, filler words, street types and ordinals.
//
// Keeping this data in a value instead of package-level maps makes the parser
// reusable across deployments: a service in Novosibirsk and a service in Tomsk
// can run the same code with different city lists, without recompilation.
type Dictionary struct {
cities []cityAlias
filler map[string]struct{}
streetTypes map[string]string
ordinalWords map[string]string
ordinalSuffixes map[string]struct{}
}
// cityAlias is one token sequence that resolves to a city.
type cityAlias struct {
tokens []string
name string
key string
}
// NewDictionary builds a dictionary with the built-in federal vocabulary and
// the supplied cities. The input slice is copied; mutating it later does not
// affect the dictionary.
func NewDictionary(cities []City) *Dictionary {
d := &Dictionary{
filler: cloneSet(defaultFillerWords),
streetTypes: cloneMap(defaultStreetTypes),
ordinalWords: cloneMap(defaultOrdinalWords),
ordinalSuffixes: cloneSet(defaultOrdinalSuffixes),
}
seen := make(map[string]struct{})
for _, c := range cities {
name := strings.TrimSpace(c.Name)
cityKey := NormalizeText(name)
if name == "" || cityKey == "" {
continue
}
aliases := append([]string{name}, c.Aliases...)
for _, alias := range aliases {
tokens := strings.Fields(NormalizeText(alias))
if len(tokens) == 0 {
continue
}
dedup := cityKey + "|" + strings.Join(tokens, " ")
if _, ok := seen[dedup]; ok {
continue
}
seen[dedup] = struct{}{}
d.cities = append(d.cities, cityAlias{
tokens: tokens,
name: name,
key: cityKey,
})
}
}
// Longest aliases win, so "новосибирская область" is preferred over a
// hypothetical shorter "новосибирская". The key is the tie-breaker that
// keeps the order deterministic when two cities share an alias.
d.sortCities()
return d
}
// sortCities orders aliases by specificity: more tokens first, then longer
// text, then key. The order decides which city wins when aliases overlap.
func (d *Dictionary) sortCities() {
sort.SliceStable(d.cities, func(i, j int) bool {
a, b := d.cities[i], d.cities[j]
if len(a.tokens) != len(b.tokens) {
return len(a.tokens) > len(b.tokens)
}
as, bs := strings.Join(a.tokens, " "), strings.Join(b.tokens, " ")
if len(as) != len(bs) {
return len(as) > len(bs)
}
if a.key != b.key {
return a.key < b.key
}
return as < bs
})
}
// withDefaultCity returns a dictionary that also recognises name as a city.
// It lets a single-city deployment use "-city Новосибирск" (or default_city)
// without duplicating the canonical name in the aliases list.
func (d *Dictionary) withDefaultCity(name string) *Dictionary {
key := NormalizeText(name)
if key == "" {
return d
}
for _, a := range d.cities {
if a.key == key {
return d
}
}
out := *d
out.cities = append(append([]cityAlias(nil), d.cities...), cityAlias{
tokens: strings.Fields(key),
name: name,
key: key,
})
out.sortCities()
return &out
}
// DefaultDictionary returns the built-in vocabulary without cities. It is the
// parser used by address.Parse for callers that do not need city awareness.
func DefaultDictionary() *Dictionary {
return NewDictionary(nil)
}
// MatchCity reports whether query consists only of a city name/alias (plus
// generic filler such as "г.") and returns the canonical city. It is the
// city-only counterpart of Parse, useful for projects that normalize city
// names without a street.
func (d *Dictionary) MatchCity(query string) (City, bool) {
name, _, rest, ok := d.matchCity(strings.Fields(NormalizeText(query)))
if !ok {
return City{}, false
}
for _, tok := range rest {
if !d.isFiller(tok) {
return City{}, false
}
}
return City{Name: name}, true
}
// matchCity finds the longest city alias in tokens and returns the city plus
// the tokens with the alias removed. Matching is contiguous on token
// boundaries, so "нск" never matches inside another word.
func (d *Dictionary) matchCity(tokens []string) (name, key string, rest []string, ok bool) {
for _, alias := range d.cities {
if idx := indexTokens(tokens, alias.tokens); idx >= 0 {
rest = make([]string, 0, len(tokens)-len(alias.tokens))
rest = append(rest, tokens[:idx]...)
rest = append(rest, tokens[idx+len(alias.tokens):]...)
return alias.name, alias.key, rest, true
}
}
return "", "", tokens, false
}
// indexTokens returns the start of the first contiguous occurrence of want in
// tokens, or -1.
func indexTokens(tokens, want []string) int {
if len(want) == 0 || len(want) > len(tokens) {
return -1
}
for i := 0; i+len(want) <= len(tokens); i++ {
matched := true
for j := range want {
if tokens[i+j] != want[j] {
matched = false
break
}
}
if matched {
return i
}
}
return -1
}
func (d *Dictionary) isFiller(tok string) bool {
_, ok := d.filler[tok]
return ok
}
func (d *Dictionary) streetType(tok string) (string, bool) {
t, ok := d.streetTypes[tok]
return t, ok
}
func (d *Dictionary) ordinalWord(tok string) (string, bool) {
o, ok := d.ordinalWords[tok]
return o, ok
}
func (d *Dictionary) isOrdinalSuffix(tok string) bool {
_, ok := d.ordinalSuffixes[tok]
return ok
}
func cloneSet(src map[string]bool) map[string]struct{} {
dst := make(map[string]struct{}, len(src))
for k, v := range src {
if v {
dst[k] = struct{}{}
}
}
return dst
}
func cloneMap(src map[string]string) map[string]string {
dst := make(map[string]string, len(src))
for k, v := range src {
dst[k] = v
}
return dst
}
// defaultFillerWords are tokens that carry no street-name meaning and that
// people routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
//
// City names are deliberately NOT listed here: they come from Dictionary, so
// the same code can serve any city (or many cities) without edits.
var defaultFillerWords = map[string]bool{
"город": true, "г": true,
"область": true, "обл": true,
"район": true, "дом": true, "д": true,
}
// defaultStreetTypes maps a surface form (including common abbreviations) to a
// single canonical street type. The type is informational for matching (we
// strip it from the key) but useful for disambiguation and display.
var defaultStreetTypes = map[string]string{
"ул": "улица", "улица": "улица",
"пер": "переулок", "пер-к": "переулок", "переулок": "переулок",
"пр": "проезд", "проезд": "проезд",
"пр-т": "проспект", "просп": "проспект", "проспект": "проспект",
"ш": "шоссе", "шоссе": "шоссе",
"мкр": "микрорайон", "микрорайон": "микрорайон",
"туп": "тупик", "тупик": "тупик",
"спуск": "спуск",
"б-р": "бульвар", "бул": "бульвар", "бульвар": "бульвар",
"пл": "площадь", "площадь": "площадь",
"пос": "поселок", "поселок": "поселок",
"тракт": "тракт",
"аллея": "аллея",
"наб": "набережная", "набережная": "набережная",
"кольцо": "кольцо",
"просека": "просека",
"квартал": "квартал",
"территория": "территория",
"дорога": "дорога",
"разъезд": "разъезд",
}
// defaultOrdinalWords maps spelled-out ordinals to their digit form. Only the
// forms that actually occur in practice are listed; unknown words are treated
// as part of the street name.
var defaultOrdinalWords = map[string]string{
"первый": "1", "первая": "1", "первое": "1",
"второй": "2", "вторая": "2", "второе": "2",
"третий": "3", "третья": "3", "третье": "3",
"четвертый": "4", "четвертая": "4", "четвертое": "4",
"пятый": "5", "пятая": "5", "пятое": "5",
"шестой": "6", "шестая": "6", "шестое": "6",
"седьмой": "7", "седьмая": "7", "седьмое": "7",
"восьмой": "8", "восьмая": "8", "восьмое": "8",
"девятый": "9", "девятая": "9", "девятое": "9",
"десятый": "10", "десятая": "10", "десятое": "10",
}
// defaultOrdinalSuffixes are the gender/number endings that may follow an
// ordinal digit when it is written as a separate token ("2 я", "10 й").
var defaultOrdinalSuffixes = map[string]bool{
"я": true, "й": true, "е": true,
"ая": true, "ый": true, "ой": true, "ое": true,
"ий": true, "ья": true, "ье": true, "ьи": true,
}
+1 -1
View File
@@ -14,7 +14,7 @@ import (
var fold = cases.Fold()
// reTrailingParenthetical matches a trailing "(...)" group, e.g. the district
// suffix that the outages portal appends: "Медкадры улица, 1 (Заельцовский
// suffix that source systems append: "Медкадры улица, 1 (Заельцовский
// район)". Such administrative noise must not be treated as part of the house.
var reTrailingParenthetical = regexp.MustCompile(`\s*\([^)]*\)\s*$`)
+113 -34
View File
@@ -24,42 +24,121 @@ var (
// Trailing apartment part: " кв 5", " квартира 12", " кв. 7а". Removed
// before house parsing so it cannot leak into the street name.
reApartment = regexp.MustCompile(`(?:^|\s)(?:квартира|кв)\.?(?:\s+\d+[а-я]?)?\s*$`)
// Spoken fractions: "18 дробь 1" is the same house unit as "18/1".
reFractionWord = regexp.MustCompile(`\s+(?:дробь|дробью)\s+`)
)
// Parse decomposes a free-form address into its structured components. It is
// total: unparseable input still yields a non-nil ParsedAddress whose street
// part is best-effort and whose HasHouse is false.
// Parser turns free-form address strings into structured addresses using a
// deployment-specific Dictionary. It is immutable and safe for concurrent use.
type Parser struct {
dict *Dictionary
defaultCityName string
defaultCityKey string
}
// NewParser builds a parser. defaultCity is used when the query does not name a
// city and the dictionary has no match; pass an empty string for deployments
// where the city must be present in every query.
func NewParser(dict *Dictionary, defaultCity string) *Parser {
if dict == nil {
dict = DefaultDictionary()
}
p := &Parser{}
if name := strings.TrimSpace(defaultCity); name != "" {
dict = dict.withDefaultCity(name)
p.defaultCityName = name
p.defaultCityKey = NormalizeText(name)
}
p.dict = dict
return p
}
// defaultParser is used by the package-level Parse convenience function. It has
// no cities and is therefore fully backward compatible: callers that need city
// awareness construct their own Parser.
var defaultParser = NewParser(DefaultDictionary(), "")
// DefaultParser returns the shared city-less parser.
func DefaultParser() *Parser {
return defaultParser
}
// Parse decomposes a free-form address with the default (city-less) parser.
func Parse(raw string) ParsedAddress {
return defaultParser.Parse(raw)
}
// Parse decomposes a free-form address into its structured components. It is
// total: unparseable input still yields a usable ParsedAddress whose street
// part is best-effort and whose HasHouse is false.
func (p *Parser) Parse(raw string) ParsedAddress {
return p.parse(raw, "")
}
// ParseInCity parses raw and forces cityName as the result's city. It is used
// by the seed pipeline, where the city is an explicit field of the source row
// and must win over whatever the address text contains.
func (p *Parser) ParseInCity(raw, cityName string) ParsedAddress {
return p.parse(raw, cityName)
}
// MatchCity reports whether query is a city name/alias and returns its
// canonical form. It does not guess a city from an arbitrary address.
func (p *Parser) MatchCity(query string) (City, bool) {
return p.dict.MatchCity(query)
}
func (p *Parser) parse(raw, forcedCity string) ParsedAddress {
normalized := NormalizeText(raw)
// A trailing apartment ("кв 5") is not part of the address identity.
normalized = strings.TrimSpace(reApartment.ReplaceAllString(normalized, ""))
p := ParsedAddress{Raw: raw, Normalized: normalized}
// Collapse the spoken fraction marker to the canonical separator before
// the right-to-left house parser runs: "18 дробь 1" -> "18/1".
normalized = reFractionWord.ReplaceAllString(normalized, "/")
parsed := ParsedAddress{Raw: raw, Normalized: normalized}
rest := normalized
if cityName, cityKey, restTokens, ok := p.dict.matchCity(strings.Fields(rest)); ok {
parsed.City = cityName
parsed.CityKey = cityKey
rest = strings.Join(restTokens, " ")
}
if forcedCity = strings.TrimSpace(forcedCity); forcedCity != "" {
parsed.City = forcedCity
parsed.CityKey = NormalizeText(forcedCity)
} else if parsed.City == "" && p.defaultCityName != "" {
// A single-city deployment can omit the city in queries. In a
// multi-city deployment default_city should stay empty: guessing a
// city is exactly the kind of silent error this service avoids.
parsed.City = p.defaultCityName
parsed.CityKey = p.defaultCityKey
}
if g, start, ok := matchSuffix(reStructure, rest); ok {
p.Structure = trimLeadingZeros(g[1])
parsed.Structure = trimLeadingZeros(g[1])
rest = strings.TrimSpace(rest[:start])
}
if g, start, ok := matchSuffix(reBuilding, rest); ok {
// The regex also matches a bare "к"; only accept it when a digit or a
// letter actually follows, otherwise it is not a corpus marker.
if g[1] != "" || g[2] != "" {
p.Building = trimLeadingZeros(g[1])
p.BuildingLetter = firstNonEmpty(g[2], g[4])
p.BuildingFrac = trimLeadingZeros(g[3])
parsed.Building = trimLeadingZeros(g[1])
parsed.BuildingLetter = firstNonEmpty(g[2], g[4])
parsed.BuildingFrac = trimLeadingZeros(g[3])
rest = strings.TrimSpace(rest[:start])
}
}
if g, start, ok := matchSuffix(reHouse, rest); ok {
p.House = trimLeadingZeros(g[1])
p.HouseLetter = firstNonEmpty(g[2], g[4])
p.HouseFrac = trimLeadingZeros(g[3])
p.HasHouse = true
parsed.House = trimLeadingZeros(g[1])
parsed.HouseLetter = firstNonEmpty(g[2], g[4])
parsed.HouseFrac = trimLeadingZeros(g[3])
parsed.HasHouse = true
rest = strings.TrimSpace(rest[:start])
}
parseStreet(&p, rest)
return p
p.parseStreet(&parsed, rest)
return parsed
}
// firstNonEmpty returns the first non-empty argument. House/building letters
@@ -73,8 +152,8 @@ func firstNonEmpty(values ...string) string {
return ""
}
// parseStreet fills the street-related fields of p from the leftover string
// (the address with its house block removed).
// parseStreet fills the street-related fields of out from the leftover string
// (the address with its city and house block removed).
//
// All recognised type tokens are removed from the name. This matters for
// registry rows such as "Гусинобродский Тракт улица 21", where both "Тракт"
@@ -82,7 +161,7 @@ func firstNonEmpty(values ...string) string {
// says "Гусинобродский 21". The first type token becomes the canonical type.
// If removing them would leave the name empty (a street literally called
// "Набережная"), the type token is kept as the name.
func parseStreet(p *ParsedAddress, s string) {
func (p *Parser) parseStreet(out *ParsedAddress, s string) {
tokens := strings.Fields(s)
name := make([]string, 0, len(tokens))
streetType := ""
@@ -90,10 +169,10 @@ func parseStreet(p *ParsedAddress, s string) {
for _, tok := range tokens {
// Filler tokens ("г", "город", "дом", "область", ...) never belong to
// the street name.
if fillerWords[tok] {
if p.dict.isFiller(tok) {
continue
}
if t, ok := streetTypes[tok]; ok {
if t, ok := p.dict.streetType(tok); ok {
if streetType == "" {
streetType = t
}
@@ -102,55 +181,55 @@ func parseStreet(p *ParsedAddress, s string) {
name = append(name, tok)
}
p.StreetType = streetType
out.StreetType = streetType
if len(name) == 0 && streetType != "" {
name = []string{streetType}
}
if len(name) == 0 {
p.StreetKey = streetKey("", "")
out.StreetKey = streetKey(out.CityKey, "", "")
return
}
name = extractOrdinal(p, name)
p.NameDisplay = strings.Join(name, " ")
name = p.extractOrdinal(out, name)
out.NameDisplay = strings.Join(name, " ")
sorted := append([]string(nil), name...)
sort.Strings(sorted)
p.NameKey = strings.Join(sorted, " ")
p.StreetKey = streetKey(p.Ordinal, p.NameKey)
out.NameKey = strings.Join(sorted, " ")
out.StreetKey = streetKey(out.CityKey, out.Ordinal, out.NameKey)
}
// extractOrdinal recognises an ordinal and removes it from the name tokens.
// It handles both the usual leading form ("2-я Сухарная") and an ordinal that
// appears after the name ("Сухарная 2-я улица"), which both people and models
// produce.
func extractOrdinal(p *ParsedAddress, tokens []string) []string {
func (p *Parser) extractOrdinal(out *ParsedAddress, tokens []string) []string {
if len(tokens) == 0 {
return tokens
}
// Leading forms first: they are by far the most common.
if m := reOrdinalToken.FindStringSubmatch(tokens[0]); m != nil {
p.Ordinal = normalizeOrdinal(m[1])
out.Ordinal = normalizeOrdinal(m[1])
return tokens[1:]
}
if isDigits(tokens[0]) && len(tokens) >= 2 && ordinalSuffixes[tokens[1]] {
p.Ordinal = normalizeOrdinal(tokens[0])
if isDigits(tokens[0]) && len(tokens) >= 2 && p.dict.isOrdinalSuffix(tokens[1]) {
out.Ordinal = normalizeOrdinal(tokens[0])
return tokens[2:]
}
if d, ok := ordinalWords[tokens[0]]; ok {
p.Ordinal = normalizeOrdinal(d)
if d, ok := p.dict.ordinalWord(tokens[0]); ok {
out.Ordinal = normalizeOrdinal(d)
return tokens[1:]
}
// Otherwise look for an ordinal anywhere among the name tokens.
for i, tok := range tokens {
if m := reOrdinalToken.FindStringSubmatch(tok); m != nil {
p.Ordinal = normalizeOrdinal(m[1])
out.Ordinal = normalizeOrdinal(m[1])
return removeRange(tokens, i, i+1)
}
if isDigits(tok) && i+1 < len(tokens) && ordinalSuffixes[tokens[i+1]] {
p.Ordinal = normalizeOrdinal(tok)
if isDigits(tok) && i+1 < len(tokens) && p.dict.isOrdinalSuffix(tokens[i+1]) {
out.Ordinal = normalizeOrdinal(tok)
return removeRange(tokens, i, i+2)
}
}
+7 -1
View File
@@ -71,6 +71,12 @@ func TestParseHouseBlock(t *testing.T) {
wantName: "выборная", wantType: "улица",
wantHouse: "45", wantLetter: "а", wantFrac: "4", wantHas: true,
},
{
name: "spoken fraction word",
raw: "Горская улица 18 дробь 1",
wantName: "горская", wantType: "улица",
wantHouse: "18", wantFrac: "1", wantHas: true,
},
{
name: "corpus letter without digit",
raw: "1-я Ельцовка улица 1 кД",
@@ -96,7 +102,7 @@ func TestParseHouseBlock(t *testing.T) {
wantHouse: "11", wantFrac: "1", wantHas: true,
},
{
name: "portal district suffix in parentheses",
name: "district suffix in parentheses",
raw: "Медкадры улица, 1 (Заельцовский район)",
wantName: "медкадры", wantType: "улица",
wantHouse: "1", wantHas: true,
+13 -67
View File
@@ -2,73 +2,19 @@ package address
import "strings"
// fillerWords are tokens that carry no street-name meaning and that people
// routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
// Dropping them makes the service tolerant of natural phrasing. District
// adjectives are deliberately NOT dropped: they can legitimately be part of a
// street name.
var fillerWords = map[string]bool{
"новосибирск": true, "нск": true, "нсо": true,
"город": true, "г": true,
"область": true, "обл": true,
"район": true, "дом": true, "д": true,
}
// streetTypes maps a surface form (including common abbreviations) to a single
// canonical street type. The type is informational for matching (we strip it
// from the key) but useful for disambiguation and display.
var streetTypes = map[string]string{
"ул": "улица", "улица": "улица",
"пер": "переулок", "пер-к": "переулок", "переулок": "переулок",
"пр": "проезд", "проезд": "проезд",
"пр-т": "проспект", "просп": "проспект", "проспект": "проспект",
"ш": "шоссе", "шоссе": "шоссе",
"мкр": "микрорайон", "микрорайон": "микрорайон",
"туп": "тупик", "тупик": "тупик",
"спуск": "спуск",
"б-р": "бульвар", "бул": "бульвар", "бульвар": "бульвар",
"пл": "площадь", "площадь": "площадь",
"пос": "поселок", "поселок": "поселок",
"тракт": "тракт",
"аллея": "аллея",
"наб": "набережная", "набережная": "набережная",
"кольцо": "кольцо",
"просека": "просека",
"квартал": "квартал",
"территория": "территория",
"дорога": "дорога",
"разъезд": "разъезд",
}
// ordinalWords maps spelled-out ordinals to their digit form. Only the forms
// that actually occur in practice are listed; unknown words are treated as
// part of the street name.
var ordinalWords = map[string]string{"первый": "1", "первая": "1", "первое": "1",
"второй": "2", "вторая": "2", "второе": "2",
"третий": "3", "третья": "3", "третье": "3",
"четвертый": "4", "четвертая": "4", "четвертое": "4",
"пятый": "5", "пятая": "5", "пятое": "5",
"шестой": "6", "шестая": "6", "шестое": "6",
"седьмой": "7", "седьмая": "7", "седьмое": "7",
"восьмой": "8", "восьмая": "8", "восьмое": "8",
"девятый": "9", "девятая": "9", "девятое": "9",
"десятый": "10", "десятая": "10", "десятое": "10",
}
// ordinalSuffixes are the gender/number endings that may follow an ordinal
// digit when it is written as a separate token ("2 я", "10 й").
var ordinalSuffixes = map[string]bool{
"я": true, "й": true, "е": true,
"ая": true, "ый": true, "ой": true, "ое": true,
"ий": true, "ья": true, "ье": true, "ьи": true,
}
// streetKey builds the canonical, type-independent street key:
// "<ordinal>|<sorted name tokens>". The type word is dropped so that a user
// who omits it ("станционная 1" instead of "Станционная улица 1") still
// matches; type confusion is surfaced later as ambiguity.
func streetKey(ordinal, nameKey string) string {
return ordinal + "|" + nameKey
// streetKey builds the canonical, city-aware, type-independent street key:
//
// "<cityKey>|<ordinal>|<sorted name tokens>"
//
// The city is part of the key so that "Ленина" in Novosibirsk and "Ленина" in
// Berdsk are different streets. The type word is dropped so that a user who
// omits it ("станционная 1" instead of "Станционная улица 1") still matches;
// type confusion is surfaced later as ambiguity.
//
// None of the components may contain "|": NormalizeText folds all punctuation
// that could appear in a name into spaces, and ordinals are digits only.
func streetKey(cityKey, ordinal, nameKey string) string {
return cityKey + "|" + ordinal + "|" + nameKey
}
// normalizeOrdinal trims leading zeros so "02-я" and "2-я" agree.
+11 -6
View File
@@ -10,7 +10,7 @@ package address
// Status is the machine-readable outcome of an address resolution.
//
// It is intentionally explicit: callers must never have to guess whether an
// empty result means "not found" or "the upstream failed".
// empty result means "not found" or "the resolver refused to choose".
type Status string
const (
@@ -25,11 +25,6 @@ const (
StatusAmbiguous Status = "ambiguous"
// StatusNotFound means nothing in the registry matched.
StatusNotFound Status = "not_found"
// StatusPortalTimeout means the external outages portal did not answer in
// time. It is an infrastructure outcome, not "no outages".
StatusPortalTimeout Status = "portal_timeout"
// StatusInfra is a generic terminal infrastructure failure.
StatusInfra Status = "infra_error"
)
// ParsedAddress is the structured form of a free-form address string.
@@ -41,6 +36,13 @@ type ParsedAddress struct {
Raw string `json:"raw"`
Normalized string `json:"normalized"`
// City is the canonical city name when the query names one (or when the
// parser is configured with a default city). CityKey is its normalized
// form. The city is part of StreetKey, so identical street names in
// different cities never collide.
City string `json:"city,omitempty"`
CityKey string `json:"city_key,omitempty"`
// Street components.
//
// NameKey is the order-insensitive canonical name (tokens sorted), so
@@ -73,6 +75,9 @@ type Entry struct {
Address string `json:"address"`
District string `json:"district,omitempty"`
City string `json:"city,omitempty"`
CityKey string `json:"city_key,omitempty"`
Ordinal string `json:"ordinal,omitempty"`
NameKey string `json:"name_key,omitempty"`
NameDisplay string `json:"name_display,omitempty"`