refactor!: make addrkit a multi-city address normalizer
- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
Maksim Totmin
@@ -0,0 +1,284 @@
|
||||
package address
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// City is one entry of the configurable city dictionary.
|
||||
//
|
||||
// Aliases are additional surface forms that refer to the city: common
|
||||
// abbreviations ("нск"), the region name ("новосибирская область") and so on.
|
||||
// The canonical Name is always accepted as an alias too. Matching is
|
||||
// token-based, case-insensitive and "ё"-insensitive, so aliases should be
|
||||
// written naturally.
|
||||
type City struct {
|
||||
Name string `json:"name"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
}
|
||||
|
||||
// Dictionary is the immutable, deployment-specific vocabulary of the parser:
|
||||
// cities, filler words, street types and ordinals.
|
||||
//
|
||||
// Keeping this data in a value instead of package-level maps makes the parser
|
||||
// reusable across deployments: a service in Novosibirsk and a service in Tomsk
|
||||
// can run the same code with different city lists, without recompilation.
|
||||
type Dictionary struct {
|
||||
cities []cityAlias
|
||||
|
||||
filler map[string]struct{}
|
||||
streetTypes map[string]string
|
||||
ordinalWords map[string]string
|
||||
ordinalSuffixes map[string]struct{}
|
||||
}
|
||||
|
||||
// cityAlias is one token sequence that resolves to a city.
|
||||
type cityAlias struct {
|
||||
tokens []string
|
||||
name string
|
||||
key string
|
||||
}
|
||||
|
||||
// NewDictionary builds a dictionary with the built-in federal vocabulary and
|
||||
// the supplied cities. The input slice is copied; mutating it later does not
|
||||
// affect the dictionary.
|
||||
func NewDictionary(cities []City) *Dictionary {
|
||||
d := &Dictionary{
|
||||
filler: cloneSet(defaultFillerWords),
|
||||
streetTypes: cloneMap(defaultStreetTypes),
|
||||
ordinalWords: cloneMap(defaultOrdinalWords),
|
||||
ordinalSuffixes: cloneSet(defaultOrdinalSuffixes),
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
for _, c := range cities {
|
||||
name := strings.TrimSpace(c.Name)
|
||||
cityKey := NormalizeText(name)
|
||||
if name == "" || cityKey == "" {
|
||||
continue
|
||||
}
|
||||
aliases := append([]string{name}, c.Aliases...)
|
||||
for _, alias := range aliases {
|
||||
tokens := strings.Fields(NormalizeText(alias))
|
||||
if len(tokens) == 0 {
|
||||
continue
|
||||
}
|
||||
dedup := cityKey + "|" + strings.Join(tokens, " ")
|
||||
if _, ok := seen[dedup]; ok {
|
||||
continue
|
||||
}
|
||||
seen[dedup] = struct{}{}
|
||||
d.cities = append(d.cities, cityAlias{
|
||||
tokens: tokens,
|
||||
name: name,
|
||||
key: cityKey,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Longest aliases win, so "новосибирская область" is preferred over a
|
||||
// hypothetical shorter "новосибирская". The key is the tie-breaker that
|
||||
// keeps the order deterministic when two cities share an alias.
|
||||
d.sortCities()
|
||||
return d
|
||||
}
|
||||
|
||||
// sortCities orders aliases by specificity: more tokens first, then longer
|
||||
// text, then key. The order decides which city wins when aliases overlap.
|
||||
func (d *Dictionary) sortCities() {
|
||||
sort.SliceStable(d.cities, func(i, j int) bool {
|
||||
a, b := d.cities[i], d.cities[j]
|
||||
if len(a.tokens) != len(b.tokens) {
|
||||
return len(a.tokens) > len(b.tokens)
|
||||
}
|
||||
as, bs := strings.Join(a.tokens, " "), strings.Join(b.tokens, " ")
|
||||
if len(as) != len(bs) {
|
||||
return len(as) > len(bs)
|
||||
}
|
||||
if a.key != b.key {
|
||||
return a.key < b.key
|
||||
}
|
||||
return as < bs
|
||||
})
|
||||
}
|
||||
|
||||
// withDefaultCity returns a dictionary that also recognises name as a city.
|
||||
// It lets a single-city deployment use "-city Новосибирск" (or default_city)
|
||||
// without duplicating the canonical name in the aliases list.
|
||||
func (d *Dictionary) withDefaultCity(name string) *Dictionary {
|
||||
key := NormalizeText(name)
|
||||
if key == "" {
|
||||
return d
|
||||
}
|
||||
for _, a := range d.cities {
|
||||
if a.key == key {
|
||||
return d
|
||||
}
|
||||
}
|
||||
out := *d
|
||||
out.cities = append(append([]cityAlias(nil), d.cities...), cityAlias{
|
||||
tokens: strings.Fields(key),
|
||||
name: name,
|
||||
key: key,
|
||||
})
|
||||
out.sortCities()
|
||||
return &out
|
||||
}
|
||||
|
||||
// DefaultDictionary returns the built-in vocabulary without cities. It is the
|
||||
// parser used by address.Parse for callers that do not need city awareness.
|
||||
func DefaultDictionary() *Dictionary {
|
||||
return NewDictionary(nil)
|
||||
}
|
||||
|
||||
// MatchCity reports whether query consists only of a city name/alias (plus
|
||||
// generic filler such as "г.") and returns the canonical city. It is the
|
||||
// city-only counterpart of Parse, useful for projects that normalize city
|
||||
// names without a street.
|
||||
func (d *Dictionary) MatchCity(query string) (City, bool) {
|
||||
name, _, rest, ok := d.matchCity(strings.Fields(NormalizeText(query)))
|
||||
if !ok {
|
||||
return City{}, false
|
||||
}
|
||||
for _, tok := range rest {
|
||||
if !d.isFiller(tok) {
|
||||
return City{}, false
|
||||
}
|
||||
}
|
||||
return City{Name: name}, true
|
||||
}
|
||||
|
||||
// matchCity finds the longest city alias in tokens and returns the city plus
|
||||
// the tokens with the alias removed. Matching is contiguous on token
|
||||
// boundaries, so "нск" never matches inside another word.
|
||||
func (d *Dictionary) matchCity(tokens []string) (name, key string, rest []string, ok bool) {
|
||||
for _, alias := range d.cities {
|
||||
if idx := indexTokens(tokens, alias.tokens); idx >= 0 {
|
||||
rest = make([]string, 0, len(tokens)-len(alias.tokens))
|
||||
rest = append(rest, tokens[:idx]...)
|
||||
rest = append(rest, tokens[idx+len(alias.tokens):]...)
|
||||
return alias.name, alias.key, rest, true
|
||||
}
|
||||
}
|
||||
return "", "", tokens, false
|
||||
}
|
||||
|
||||
// indexTokens returns the start of the first contiguous occurrence of want in
|
||||
// tokens, or -1.
|
||||
func indexTokens(tokens, want []string) int {
|
||||
if len(want) == 0 || len(want) > len(tokens) {
|
||||
return -1
|
||||
}
|
||||
for i := 0; i+len(want) <= len(tokens); i++ {
|
||||
matched := true
|
||||
for j := range want {
|
||||
if tokens[i+j] != want[j] {
|
||||
matched = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (d *Dictionary) isFiller(tok string) bool {
|
||||
_, ok := d.filler[tok]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (d *Dictionary) streetType(tok string) (string, bool) {
|
||||
t, ok := d.streetTypes[tok]
|
||||
return t, ok
|
||||
}
|
||||
|
||||
func (d *Dictionary) ordinalWord(tok string) (string, bool) {
|
||||
o, ok := d.ordinalWords[tok]
|
||||
return o, ok
|
||||
}
|
||||
|
||||
func (d *Dictionary) isOrdinalSuffix(tok string) bool {
|
||||
_, ok := d.ordinalSuffixes[tok]
|
||||
return ok
|
||||
}
|
||||
|
||||
func cloneSet(src map[string]bool) map[string]struct{} {
|
||||
dst := make(map[string]struct{}, len(src))
|
||||
for k, v := range src {
|
||||
if v {
|
||||
dst[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func cloneMap(src map[string]string) map[string]string {
|
||||
dst := make(map[string]string, len(src))
|
||||
for k, v := range src {
|
||||
dst[k] = v
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// defaultFillerWords are tokens that carry no street-name meaning and that
|
||||
// people routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
|
||||
//
|
||||
// City names are deliberately NOT listed here: they come from Dictionary, so
|
||||
// the same code can serve any city (or many cities) without edits.
|
||||
var defaultFillerWords = map[string]bool{
|
||||
"город": true, "г": true,
|
||||
"область": true, "обл": true,
|
||||
"район": true, "дом": true, "д": true,
|
||||
}
|
||||
|
||||
// defaultStreetTypes maps a surface form (including common abbreviations) to a
|
||||
// single canonical street type. The type is informational for matching (we
|
||||
// strip it from the key) but useful for disambiguation and display.
|
||||
var defaultStreetTypes = map[string]string{
|
||||
"ул": "улица", "улица": "улица",
|
||||
"пер": "переулок", "пер-к": "переулок", "переулок": "переулок",
|
||||
"пр": "проезд", "проезд": "проезд",
|
||||
"пр-т": "проспект", "просп": "проспект", "проспект": "проспект",
|
||||
"ш": "шоссе", "шоссе": "шоссе",
|
||||
"мкр": "микрорайон", "микрорайон": "микрорайон",
|
||||
"туп": "тупик", "тупик": "тупик",
|
||||
"спуск": "спуск",
|
||||
"б-р": "бульвар", "бул": "бульвар", "бульвар": "бульвар",
|
||||
"пл": "площадь", "площадь": "площадь",
|
||||
"пос": "поселок", "поселок": "поселок",
|
||||
"тракт": "тракт",
|
||||
"аллея": "аллея",
|
||||
"наб": "набережная", "набережная": "набережная",
|
||||
"кольцо": "кольцо",
|
||||
"просека": "просека",
|
||||
"квартал": "квартал",
|
||||
"территория": "территория",
|
||||
"дорога": "дорога",
|
||||
"разъезд": "разъезд",
|
||||
}
|
||||
|
||||
// defaultOrdinalWords maps spelled-out ordinals to their digit form. Only the
|
||||
// forms that actually occur in practice are listed; unknown words are treated
|
||||
// as part of the street name.
|
||||
var defaultOrdinalWords = map[string]string{
|
||||
"первый": "1", "первая": "1", "первое": "1",
|
||||
"второй": "2", "вторая": "2", "второе": "2",
|
||||
"третий": "3", "третья": "3", "третье": "3",
|
||||
"четвертый": "4", "четвертая": "4", "четвертое": "4",
|
||||
"пятый": "5", "пятая": "5", "пятое": "5",
|
||||
"шестой": "6", "шестая": "6", "шестое": "6",
|
||||
"седьмой": "7", "седьмая": "7", "седьмое": "7",
|
||||
"восьмой": "8", "восьмая": "8", "восьмое": "8",
|
||||
"девятый": "9", "девятая": "9", "девятое": "9",
|
||||
"десятый": "10", "десятая": "10", "десятое": "10",
|
||||
}
|
||||
|
||||
// defaultOrdinalSuffixes are the gender/number endings that may follow an
|
||||
// ordinal digit when it is written as a separate token ("2 я", "10 й").
|
||||
var defaultOrdinalSuffixes = map[string]bool{
|
||||
"я": true, "й": true, "е": true,
|
||||
"ая": true, "ый": true, "ой": true, "ое": true,
|
||||
"ий": true, "ья": true, "ье": true, "ьи": true,
|
||||
}
|
||||
Reference in New Issue
Block a user