refactor!: make addrkit a multi-city address normalizer
- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
package address
|
||||
|
||||
import "testing"
|
||||
|
||||
func testDictionary() *Dictionary {
|
||||
return NewDictionary([]City{
|
||||
{Name: "Новосибирск", Aliases: []string{"нск", "новосиб", "новосибирская область", "нсо"}},
|
||||
{Name: "Бердск"},
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseCityFromAliases(t *testing.T) {
|
||||
parser := NewParser(testDictionary(), "")
|
||||
|
||||
tests := []struct {
|
||||
raw string
|
||||
city string
|
||||
nameKey string
|
||||
}{
|
||||
{"нск, 2-я Станционная улица 1", "Новосибирск", "станционная"},
|
||||
{"НСК Ленина 5", "Новосибирск", "ленина"},
|
||||
{"Новосибирская область, Станционная 1", "Новосибирск", "станционная"},
|
||||
{"Бердск, Ленина 5", "Бердск", "ленина"},
|
||||
{"г. Новосибирск, ул. Ленина 5", "Новосибирск", "ленина"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := parser.Parse(tt.raw)
|
||||
if got.City != tt.city {
|
||||
t.Errorf("%q: City = %q, want %q", tt.raw, got.City, tt.city)
|
||||
}
|
||||
if got.NameKey != tt.nameKey {
|
||||
t.Errorf("%q: NameKey = %q, want %q", tt.raw, got.NameKey, tt.nameKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCityIsPartOfStreetKey(t *testing.T) {
|
||||
parser := NewParser(testDictionary(), "")
|
||||
|
||||
nsk := parser.Parse("Новосибирск, Ленина 5")
|
||||
berdsk := parser.Parse("Бердск, Ленина 5")
|
||||
if nsk.StreetKey == berdsk.StreetKey {
|
||||
t.Fatalf("same street key for different cities: %q", nsk.StreetKey)
|
||||
}
|
||||
if nsk.CityKey != "новосибирск" || berdsk.CityKey != "бердск" {
|
||||
t.Fatalf("city keys = %q / %q", nsk.CityKey, berdsk.CityKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDefaultCity(t *testing.T) {
|
||||
parser := NewParser(testDictionary(), "Новосибирск")
|
||||
got := parser.Parse("Ленина 5")
|
||||
if got.City != "Новосибирск" || got.CityKey != "новосибирск" {
|
||||
t.Fatalf("City = %q / %q", got.City, got.CityKey)
|
||||
}
|
||||
|
||||
// The default city is also recognised when the user spells it out, even
|
||||
// if the dictionary contains only aliases.
|
||||
explicit := NewParser(DefaultDictionary(), "Новосибирск").Parse("г. Новосибирск, Ленина 5")
|
||||
if explicit.City != "Новосибирск" || explicit.NameKey != "ленина" {
|
||||
t.Fatalf("explicit default city: %+v", explicit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInCityOverridesDetectedCity(t *testing.T) {
|
||||
parser := NewParser(testDictionary(), "")
|
||||
got := parser.ParseInCity("Бердск, Ленина 5", "Новосибирск")
|
||||
if got.City != "Новосибирск" {
|
||||
t.Fatalf("City = %q, want forced Новосибирск", got.City)
|
||||
}
|
||||
if got.NameKey != "ленина" {
|
||||
t.Fatalf("NameKey = %q", got.NameKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWithoutCityKeepsCityKeyEmpty(t *testing.T) {
|
||||
parser := NewParser(testDictionary(), "")
|
||||
got := parser.Parse("Ленина 5")
|
||||
if got.City != "" || got.CityKey != "" {
|
||||
t.Fatalf("City = %q / %q, want empty", got.City, got.CityKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeTextStillLeavesUnknownCityTokens(t *testing.T) {
|
||||
// A city that is not in the dictionary must not be silently dropped: the
|
||||
// resolver should fail rather than guess another city.
|
||||
parser := NewParser(testDictionary(), "")
|
||||
got := parser.Parse("Томск, Ленина 5")
|
||||
if got.City != "" {
|
||||
t.Fatalf("City = %q, want empty for unknown city", got.City)
|
||||
}
|
||||
if got.NameKey == "ленина" {
|
||||
t.Fatalf("unknown city token disappeared from the street name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchCity(t *testing.T) {
|
||||
dict := testDictionary()
|
||||
cases := []struct {
|
||||
query string
|
||||
want string
|
||||
ok bool
|
||||
}{
|
||||
{"нск", "Новосибирск", true},
|
||||
{"г. Новосибирск", "Новосибирск", true},
|
||||
{"Новосибирская область", "Новосибирск", true},
|
||||
{"Бердск", "Бердск", true},
|
||||
{"Бердск, Ленина 5", "", false},
|
||||
{"Томск", "", false},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
got, ok := dict.MatchCity(tt.query)
|
||||
if ok != tt.ok || got.Name != tt.want {
|
||||
t.Errorf("MatchCity(%q) = %+v, %v; want %q, %v", tt.query, got, ok, tt.want, tt.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzParse(f *testing.F) {
|
||||
for _, seed := range []string{
|
||||
"",
|
||||
"2-я Станционная улица 1/1 стр.",
|
||||
"нск, Ленина 5",
|
||||
"Бердск, Ленина 5 к2",
|
||||
"г. Новосибирск, ул. Кошурникова, д. 11/1 кв. 5",
|
||||
"\x00\xff",
|
||||
} {
|
||||
f.Add(seed)
|
||||
}
|
||||
parser := NewParser(testDictionary(), "")
|
||||
f.Fuzz(func(t *testing.T, raw string) {
|
||||
first := parser.Parse(raw)
|
||||
second := parser.Parse(raw)
|
||||
if first.StreetKey != second.StreetKey {
|
||||
t.Fatalf("non-deterministic street key for %q: %q vs %q", raw, first.StreetKey, second.StreetKey)
|
||||
}
|
||||
if NormalizeText(raw) != NormalizeText(NormalizeText(raw)) {
|
||||
t.Fatalf("NormalizeText is not idempotent for %q", raw)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package address
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// City is one entry of the configurable city dictionary.
|
||||
//
|
||||
// Aliases are additional surface forms that refer to the city: common
|
||||
// abbreviations ("нск"), the region name ("новосибирская область") and so on.
|
||||
// The canonical Name is always accepted as an alias too. Matching is
|
||||
// token-based, case-insensitive and "ё"-insensitive, so aliases should be
|
||||
// written naturally.
|
||||
type City struct {
|
||||
Name string `json:"name"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
}
|
||||
|
||||
// Dictionary is the immutable, deployment-specific vocabulary of the parser:
|
||||
// cities, filler words, street types and ordinals.
|
||||
//
|
||||
// Keeping this data in a value instead of package-level maps makes the parser
|
||||
// reusable across deployments: a service in Novosibirsk and a service in Tomsk
|
||||
// can run the same code with different city lists, without recompilation.
|
||||
type Dictionary struct {
|
||||
cities []cityAlias
|
||||
|
||||
filler map[string]struct{}
|
||||
streetTypes map[string]string
|
||||
ordinalWords map[string]string
|
||||
ordinalSuffixes map[string]struct{}
|
||||
}
|
||||
|
||||
// cityAlias is one token sequence that resolves to a city.
|
||||
type cityAlias struct {
|
||||
tokens []string
|
||||
name string
|
||||
key string
|
||||
}
|
||||
|
||||
// NewDictionary builds a dictionary with the built-in federal vocabulary and
|
||||
// the supplied cities. The input slice is copied; mutating it later does not
|
||||
// affect the dictionary.
|
||||
func NewDictionary(cities []City) *Dictionary {
|
||||
d := &Dictionary{
|
||||
filler: cloneSet(defaultFillerWords),
|
||||
streetTypes: cloneMap(defaultStreetTypes),
|
||||
ordinalWords: cloneMap(defaultOrdinalWords),
|
||||
ordinalSuffixes: cloneSet(defaultOrdinalSuffixes),
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
for _, c := range cities {
|
||||
name := strings.TrimSpace(c.Name)
|
||||
cityKey := NormalizeText(name)
|
||||
if name == "" || cityKey == "" {
|
||||
continue
|
||||
}
|
||||
aliases := append([]string{name}, c.Aliases...)
|
||||
for _, alias := range aliases {
|
||||
tokens := strings.Fields(NormalizeText(alias))
|
||||
if len(tokens) == 0 {
|
||||
continue
|
||||
}
|
||||
dedup := cityKey + "|" + strings.Join(tokens, " ")
|
||||
if _, ok := seen[dedup]; ok {
|
||||
continue
|
||||
}
|
||||
seen[dedup] = struct{}{}
|
||||
d.cities = append(d.cities, cityAlias{
|
||||
tokens: tokens,
|
||||
name: name,
|
||||
key: cityKey,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Longest aliases win, so "новосибирская область" is preferred over a
|
||||
// hypothetical shorter "новосибирская". The key is the tie-breaker that
|
||||
// keeps the order deterministic when two cities share an alias.
|
||||
d.sortCities()
|
||||
return d
|
||||
}
|
||||
|
||||
// sortCities orders aliases by specificity: more tokens first, then longer
|
||||
// text, then key. The order decides which city wins when aliases overlap.
|
||||
func (d *Dictionary) sortCities() {
|
||||
sort.SliceStable(d.cities, func(i, j int) bool {
|
||||
a, b := d.cities[i], d.cities[j]
|
||||
if len(a.tokens) != len(b.tokens) {
|
||||
return len(a.tokens) > len(b.tokens)
|
||||
}
|
||||
as, bs := strings.Join(a.tokens, " "), strings.Join(b.tokens, " ")
|
||||
if len(as) != len(bs) {
|
||||
return len(as) > len(bs)
|
||||
}
|
||||
if a.key != b.key {
|
||||
return a.key < b.key
|
||||
}
|
||||
return as < bs
|
||||
})
|
||||
}
|
||||
|
||||
// withDefaultCity returns a dictionary that also recognises name as a city.
|
||||
// It lets a single-city deployment use "-city Новосибирск" (or default_city)
|
||||
// without duplicating the canonical name in the aliases list.
|
||||
func (d *Dictionary) withDefaultCity(name string) *Dictionary {
|
||||
key := NormalizeText(name)
|
||||
if key == "" {
|
||||
return d
|
||||
}
|
||||
for _, a := range d.cities {
|
||||
if a.key == key {
|
||||
return d
|
||||
}
|
||||
}
|
||||
out := *d
|
||||
out.cities = append(append([]cityAlias(nil), d.cities...), cityAlias{
|
||||
tokens: strings.Fields(key),
|
||||
name: name,
|
||||
key: key,
|
||||
})
|
||||
out.sortCities()
|
||||
return &out
|
||||
}
|
||||
|
||||
// DefaultDictionary returns the built-in vocabulary without cities. It is the
|
||||
// parser used by address.Parse for callers that do not need city awareness.
|
||||
func DefaultDictionary() *Dictionary {
|
||||
return NewDictionary(nil)
|
||||
}
|
||||
|
||||
// MatchCity reports whether query consists only of a city name/alias (plus
|
||||
// generic filler such as "г.") and returns the canonical city. It is the
|
||||
// city-only counterpart of Parse, useful for projects that normalize city
|
||||
// names without a street.
|
||||
func (d *Dictionary) MatchCity(query string) (City, bool) {
|
||||
name, _, rest, ok := d.matchCity(strings.Fields(NormalizeText(query)))
|
||||
if !ok {
|
||||
return City{}, false
|
||||
}
|
||||
for _, tok := range rest {
|
||||
if !d.isFiller(tok) {
|
||||
return City{}, false
|
||||
}
|
||||
}
|
||||
return City{Name: name}, true
|
||||
}
|
||||
|
||||
// matchCity finds the longest city alias in tokens and returns the city plus
|
||||
// the tokens with the alias removed. Matching is contiguous on token
|
||||
// boundaries, so "нск" never matches inside another word.
|
||||
func (d *Dictionary) matchCity(tokens []string) (name, key string, rest []string, ok bool) {
|
||||
for _, alias := range d.cities {
|
||||
if idx := indexTokens(tokens, alias.tokens); idx >= 0 {
|
||||
rest = make([]string, 0, len(tokens)-len(alias.tokens))
|
||||
rest = append(rest, tokens[:idx]...)
|
||||
rest = append(rest, tokens[idx+len(alias.tokens):]...)
|
||||
return alias.name, alias.key, rest, true
|
||||
}
|
||||
}
|
||||
return "", "", tokens, false
|
||||
}
|
||||
|
||||
// indexTokens returns the start of the first contiguous occurrence of want in
|
||||
// tokens, or -1.
|
||||
func indexTokens(tokens, want []string) int {
|
||||
if len(want) == 0 || len(want) > len(tokens) {
|
||||
return -1
|
||||
}
|
||||
for i := 0; i+len(want) <= len(tokens); i++ {
|
||||
matched := true
|
||||
for j := range want {
|
||||
if tokens[i+j] != want[j] {
|
||||
matched = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (d *Dictionary) isFiller(tok string) bool {
|
||||
_, ok := d.filler[tok]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (d *Dictionary) streetType(tok string) (string, bool) {
|
||||
t, ok := d.streetTypes[tok]
|
||||
return t, ok
|
||||
}
|
||||
|
||||
func (d *Dictionary) ordinalWord(tok string) (string, bool) {
|
||||
o, ok := d.ordinalWords[tok]
|
||||
return o, ok
|
||||
}
|
||||
|
||||
func (d *Dictionary) isOrdinalSuffix(tok string) bool {
|
||||
_, ok := d.ordinalSuffixes[tok]
|
||||
return ok
|
||||
}
|
||||
|
||||
func cloneSet(src map[string]bool) map[string]struct{} {
|
||||
dst := make(map[string]struct{}, len(src))
|
||||
for k, v := range src {
|
||||
if v {
|
||||
dst[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func cloneMap(src map[string]string) map[string]string {
|
||||
dst := make(map[string]string, len(src))
|
||||
for k, v := range src {
|
||||
dst[k] = v
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// defaultFillerWords are tokens that carry no street-name meaning and that
|
||||
// people routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
|
||||
//
|
||||
// City names are deliberately NOT listed here: they come from Dictionary, so
|
||||
// the same code can serve any city (or many cities) without edits.
|
||||
var defaultFillerWords = map[string]bool{
|
||||
"город": true, "г": true,
|
||||
"область": true, "обл": true,
|
||||
"район": true, "дом": true, "д": true,
|
||||
}
|
||||
|
||||
// defaultStreetTypes maps a surface form (including common abbreviations) to a
|
||||
// single canonical street type. The type is informational for matching (we
|
||||
// strip it from the key) but useful for disambiguation and display.
|
||||
var defaultStreetTypes = map[string]string{
|
||||
"ул": "улица", "улица": "улица",
|
||||
"пер": "переулок", "пер-к": "переулок", "переулок": "переулок",
|
||||
"пр": "проезд", "проезд": "проезд",
|
||||
"пр-т": "проспект", "просп": "проспект", "проспект": "проспект",
|
||||
"ш": "шоссе", "шоссе": "шоссе",
|
||||
"мкр": "микрорайон", "микрорайон": "микрорайон",
|
||||
"туп": "тупик", "тупик": "тупик",
|
||||
"спуск": "спуск",
|
||||
"б-р": "бульвар", "бул": "бульвар", "бульвар": "бульвар",
|
||||
"пл": "площадь", "площадь": "площадь",
|
||||
"пос": "поселок", "поселок": "поселок",
|
||||
"тракт": "тракт",
|
||||
"аллея": "аллея",
|
||||
"наб": "набережная", "набережная": "набережная",
|
||||
"кольцо": "кольцо",
|
||||
"просека": "просека",
|
||||
"квартал": "квартал",
|
||||
"территория": "территория",
|
||||
"дорога": "дорога",
|
||||
"разъезд": "разъезд",
|
||||
}
|
||||
|
||||
// defaultOrdinalWords maps spelled-out ordinals to their digit form. Only the
|
||||
// forms that actually occur in practice are listed; unknown words are treated
|
||||
// as part of the street name.
|
||||
var defaultOrdinalWords = map[string]string{
|
||||
"первый": "1", "первая": "1", "первое": "1",
|
||||
"второй": "2", "вторая": "2", "второе": "2",
|
||||
"третий": "3", "третья": "3", "третье": "3",
|
||||
"четвертый": "4", "четвертая": "4", "четвертое": "4",
|
||||
"пятый": "5", "пятая": "5", "пятое": "5",
|
||||
"шестой": "6", "шестая": "6", "шестое": "6",
|
||||
"седьмой": "7", "седьмая": "7", "седьмое": "7",
|
||||
"восьмой": "8", "восьмая": "8", "восьмое": "8",
|
||||
"девятый": "9", "девятая": "9", "девятое": "9",
|
||||
"десятый": "10", "десятая": "10", "десятое": "10",
|
||||
}
|
||||
|
||||
// defaultOrdinalSuffixes are the gender/number endings that may follow an
|
||||
// ordinal digit when it is written as a separate token ("2 я", "10 й").
|
||||
var defaultOrdinalSuffixes = map[string]bool{
|
||||
"я": true, "й": true, "е": true,
|
||||
"ая": true, "ый": true, "ой": true, "ое": true,
|
||||
"ий": true, "ья": true, "ье": true, "ьи": true,
|
||||
}
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
var fold = cases.Fold()
|
||||
|
||||
// reTrailingParenthetical matches a trailing "(...)" group, e.g. the district
|
||||
// suffix that the outages portal appends: "Медкадры улица, 1 (Заельцовский
|
||||
// suffix that source systems append: "Медкадры улица, 1 (Заельцовский
|
||||
// район)". Such administrative noise must not be treated as part of the house.
|
||||
var reTrailingParenthetical = regexp.MustCompile(`\s*\([^)]*\)\s*$`)
|
||||
|
||||
|
||||
@@ -24,42 +24,121 @@ var (
|
||||
// Trailing apartment part: " кв 5", " квартира 12", " кв. 7а". Removed
|
||||
// before house parsing so it cannot leak into the street name.
|
||||
reApartment = regexp.MustCompile(`(?:^|\s)(?:квартира|кв)\.?(?:\s+\d+[а-я]?)?\s*$`)
|
||||
|
||||
// Spoken fractions: "18 дробь 1" is the same house unit as "18/1".
|
||||
reFractionWord = regexp.MustCompile(`\s+(?:дробь|дробью)\s+`)
|
||||
)
|
||||
|
||||
// Parse decomposes a free-form address into its structured components. It is
|
||||
// total: unparseable input still yields a non-nil ParsedAddress whose street
|
||||
// part is best-effort and whose HasHouse is false.
|
||||
// Parser turns free-form address strings into structured addresses using a
|
||||
// deployment-specific Dictionary. It is immutable and safe for concurrent use.
|
||||
type Parser struct {
|
||||
dict *Dictionary
|
||||
defaultCityName string
|
||||
defaultCityKey string
|
||||
}
|
||||
|
||||
// NewParser builds a parser. defaultCity is used when the query does not name a
|
||||
// city and the dictionary has no match; pass an empty string for deployments
|
||||
// where the city must be present in every query.
|
||||
func NewParser(dict *Dictionary, defaultCity string) *Parser {
|
||||
if dict == nil {
|
||||
dict = DefaultDictionary()
|
||||
}
|
||||
p := &Parser{}
|
||||
if name := strings.TrimSpace(defaultCity); name != "" {
|
||||
dict = dict.withDefaultCity(name)
|
||||
p.defaultCityName = name
|
||||
p.defaultCityKey = NormalizeText(name)
|
||||
}
|
||||
p.dict = dict
|
||||
return p
|
||||
}
|
||||
|
||||
// defaultParser is used by the package-level Parse convenience function. It has
|
||||
// no cities and is therefore fully backward compatible: callers that need city
|
||||
// awareness construct their own Parser.
|
||||
var defaultParser = NewParser(DefaultDictionary(), "")
|
||||
|
||||
// DefaultParser returns the shared city-less parser.
|
||||
func DefaultParser() *Parser {
|
||||
return defaultParser
|
||||
}
|
||||
|
||||
// Parse decomposes a free-form address with the default (city-less) parser.
|
||||
func Parse(raw string) ParsedAddress {
|
||||
return defaultParser.Parse(raw)
|
||||
}
|
||||
|
||||
// Parse decomposes a free-form address into its structured components. It is
|
||||
// total: unparseable input still yields a usable ParsedAddress whose street
|
||||
// part is best-effort and whose HasHouse is false.
|
||||
func (p *Parser) Parse(raw string) ParsedAddress {
|
||||
return p.parse(raw, "")
|
||||
}
|
||||
|
||||
// ParseInCity parses raw and forces cityName as the result's city. It is used
|
||||
// by the seed pipeline, where the city is an explicit field of the source row
|
||||
// and must win over whatever the address text contains.
|
||||
func (p *Parser) ParseInCity(raw, cityName string) ParsedAddress {
|
||||
return p.parse(raw, cityName)
|
||||
}
|
||||
|
||||
// MatchCity reports whether query is a city name/alias and returns its
|
||||
// canonical form. It does not guess a city from an arbitrary address.
|
||||
func (p *Parser) MatchCity(query string) (City, bool) {
|
||||
return p.dict.MatchCity(query)
|
||||
}
|
||||
|
||||
func (p *Parser) parse(raw, forcedCity string) ParsedAddress {
|
||||
normalized := NormalizeText(raw)
|
||||
// A trailing apartment ("кв 5") is not part of the address identity.
|
||||
normalized = strings.TrimSpace(reApartment.ReplaceAllString(normalized, ""))
|
||||
p := ParsedAddress{Raw: raw, Normalized: normalized}
|
||||
// Collapse the spoken fraction marker to the canonical separator before
|
||||
// the right-to-left house parser runs: "18 дробь 1" -> "18/1".
|
||||
normalized = reFractionWord.ReplaceAllString(normalized, "/")
|
||||
parsed := ParsedAddress{Raw: raw, Normalized: normalized}
|
||||
|
||||
rest := normalized
|
||||
if cityName, cityKey, restTokens, ok := p.dict.matchCity(strings.Fields(rest)); ok {
|
||||
parsed.City = cityName
|
||||
parsed.CityKey = cityKey
|
||||
rest = strings.Join(restTokens, " ")
|
||||
}
|
||||
if forcedCity = strings.TrimSpace(forcedCity); forcedCity != "" {
|
||||
parsed.City = forcedCity
|
||||
parsed.CityKey = NormalizeText(forcedCity)
|
||||
} else if parsed.City == "" && p.defaultCityName != "" {
|
||||
// A single-city deployment can omit the city in queries. In a
|
||||
// multi-city deployment default_city should stay empty: guessing a
|
||||
// city is exactly the kind of silent error this service avoids.
|
||||
parsed.City = p.defaultCityName
|
||||
parsed.CityKey = p.defaultCityKey
|
||||
}
|
||||
|
||||
if g, start, ok := matchSuffix(reStructure, rest); ok {
|
||||
p.Structure = trimLeadingZeros(g[1])
|
||||
parsed.Structure = trimLeadingZeros(g[1])
|
||||
rest = strings.TrimSpace(rest[:start])
|
||||
}
|
||||
if g, start, ok := matchSuffix(reBuilding, rest); ok {
|
||||
// The regex also matches a bare "к"; only accept it when a digit or a
|
||||
// letter actually follows, otherwise it is not a corpus marker.
|
||||
if g[1] != "" || g[2] != "" {
|
||||
p.Building = trimLeadingZeros(g[1])
|
||||
p.BuildingLetter = firstNonEmpty(g[2], g[4])
|
||||
p.BuildingFrac = trimLeadingZeros(g[3])
|
||||
parsed.Building = trimLeadingZeros(g[1])
|
||||
parsed.BuildingLetter = firstNonEmpty(g[2], g[4])
|
||||
parsed.BuildingFrac = trimLeadingZeros(g[3])
|
||||
rest = strings.TrimSpace(rest[:start])
|
||||
}
|
||||
}
|
||||
if g, start, ok := matchSuffix(reHouse, rest); ok {
|
||||
p.House = trimLeadingZeros(g[1])
|
||||
p.HouseLetter = firstNonEmpty(g[2], g[4])
|
||||
p.HouseFrac = trimLeadingZeros(g[3])
|
||||
p.HasHouse = true
|
||||
parsed.House = trimLeadingZeros(g[1])
|
||||
parsed.HouseLetter = firstNonEmpty(g[2], g[4])
|
||||
parsed.HouseFrac = trimLeadingZeros(g[3])
|
||||
parsed.HasHouse = true
|
||||
rest = strings.TrimSpace(rest[:start])
|
||||
}
|
||||
|
||||
parseStreet(&p, rest)
|
||||
return p
|
||||
p.parseStreet(&parsed, rest)
|
||||
return parsed
|
||||
}
|
||||
|
||||
// firstNonEmpty returns the first non-empty argument. House/building letters
|
||||
@@ -73,8 +152,8 @@ func firstNonEmpty(values ...string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseStreet fills the street-related fields of p from the leftover string
|
||||
// (the address with its house block removed).
|
||||
// parseStreet fills the street-related fields of out from the leftover string
|
||||
// (the address with its city and house block removed).
|
||||
//
|
||||
// All recognised type tokens are removed from the name. This matters for
|
||||
// registry rows such as "Гусинобродский Тракт улица 21", where both "Тракт"
|
||||
@@ -82,7 +161,7 @@ func firstNonEmpty(values ...string) string {
|
||||
// says "Гусинобродский 21". The first type token becomes the canonical type.
|
||||
// If removing them would leave the name empty (a street literally called
|
||||
// "Набережная"), the type token is kept as the name.
|
||||
func parseStreet(p *ParsedAddress, s string) {
|
||||
func (p *Parser) parseStreet(out *ParsedAddress, s string) {
|
||||
tokens := strings.Fields(s)
|
||||
name := make([]string, 0, len(tokens))
|
||||
streetType := ""
|
||||
@@ -90,10 +169,10 @@ func parseStreet(p *ParsedAddress, s string) {
|
||||
for _, tok := range tokens {
|
||||
// Filler tokens ("г", "город", "дом", "область", ...) never belong to
|
||||
// the street name.
|
||||
if fillerWords[tok] {
|
||||
if p.dict.isFiller(tok) {
|
||||
continue
|
||||
}
|
||||
if t, ok := streetTypes[tok]; ok {
|
||||
if t, ok := p.dict.streetType(tok); ok {
|
||||
if streetType == "" {
|
||||
streetType = t
|
||||
}
|
||||
@@ -102,55 +181,55 @@ func parseStreet(p *ParsedAddress, s string) {
|
||||
name = append(name, tok)
|
||||
}
|
||||
|
||||
p.StreetType = streetType
|
||||
out.StreetType = streetType
|
||||
if len(name) == 0 && streetType != "" {
|
||||
name = []string{streetType}
|
||||
}
|
||||
if len(name) == 0 {
|
||||
p.StreetKey = streetKey("", "")
|
||||
out.StreetKey = streetKey(out.CityKey, "", "")
|
||||
return
|
||||
}
|
||||
|
||||
name = extractOrdinal(p, name)
|
||||
p.NameDisplay = strings.Join(name, " ")
|
||||
name = p.extractOrdinal(out, name)
|
||||
out.NameDisplay = strings.Join(name, " ")
|
||||
|
||||
sorted := append([]string(nil), name...)
|
||||
sort.Strings(sorted)
|
||||
p.NameKey = strings.Join(sorted, " ")
|
||||
p.StreetKey = streetKey(p.Ordinal, p.NameKey)
|
||||
out.NameKey = strings.Join(sorted, " ")
|
||||
out.StreetKey = streetKey(out.CityKey, out.Ordinal, out.NameKey)
|
||||
}
|
||||
|
||||
// extractOrdinal recognises an ordinal and removes it from the name tokens.
|
||||
// It handles both the usual leading form ("2-я Сухарная") and an ordinal that
|
||||
// appears after the name ("Сухарная 2-я улица"), which both people and models
|
||||
// produce.
|
||||
func extractOrdinal(p *ParsedAddress, tokens []string) []string {
|
||||
func (p *Parser) extractOrdinal(out *ParsedAddress, tokens []string) []string {
|
||||
if len(tokens) == 0 {
|
||||
return tokens
|
||||
}
|
||||
|
||||
// Leading forms first: they are by far the most common.
|
||||
if m := reOrdinalToken.FindStringSubmatch(tokens[0]); m != nil {
|
||||
p.Ordinal = normalizeOrdinal(m[1])
|
||||
out.Ordinal = normalizeOrdinal(m[1])
|
||||
return tokens[1:]
|
||||
}
|
||||
if isDigits(tokens[0]) && len(tokens) >= 2 && ordinalSuffixes[tokens[1]] {
|
||||
p.Ordinal = normalizeOrdinal(tokens[0])
|
||||
if isDigits(tokens[0]) && len(tokens) >= 2 && p.dict.isOrdinalSuffix(tokens[1]) {
|
||||
out.Ordinal = normalizeOrdinal(tokens[0])
|
||||
return tokens[2:]
|
||||
}
|
||||
if d, ok := ordinalWords[tokens[0]]; ok {
|
||||
p.Ordinal = normalizeOrdinal(d)
|
||||
if d, ok := p.dict.ordinalWord(tokens[0]); ok {
|
||||
out.Ordinal = normalizeOrdinal(d)
|
||||
return tokens[1:]
|
||||
}
|
||||
|
||||
// Otherwise look for an ordinal anywhere among the name tokens.
|
||||
for i, tok := range tokens {
|
||||
if m := reOrdinalToken.FindStringSubmatch(tok); m != nil {
|
||||
p.Ordinal = normalizeOrdinal(m[1])
|
||||
out.Ordinal = normalizeOrdinal(m[1])
|
||||
return removeRange(tokens, i, i+1)
|
||||
}
|
||||
if isDigits(tok) && i+1 < len(tokens) && ordinalSuffixes[tokens[i+1]] {
|
||||
p.Ordinal = normalizeOrdinal(tok)
|
||||
if isDigits(tok) && i+1 < len(tokens) && p.dict.isOrdinalSuffix(tokens[i+1]) {
|
||||
out.Ordinal = normalizeOrdinal(tok)
|
||||
return removeRange(tokens, i, i+2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,6 +71,12 @@ func TestParseHouseBlock(t *testing.T) {
|
||||
wantName: "выборная", wantType: "улица",
|
||||
wantHouse: "45", wantLetter: "а", wantFrac: "4", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "spoken fraction word",
|
||||
raw: "Горская улица 18 дробь 1",
|
||||
wantName: "горская", wantType: "улица",
|
||||
wantHouse: "18", wantFrac: "1", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "corpus letter without digit",
|
||||
raw: "1-я Ельцовка улица 1 кД",
|
||||
@@ -96,7 +102,7 @@ func TestParseHouseBlock(t *testing.T) {
|
||||
wantHouse: "11", wantFrac: "1", wantHas: true,
|
||||
},
|
||||
{
|
||||
name: "portal district suffix in parentheses",
|
||||
name: "district suffix in parentheses",
|
||||
raw: "Медкадры улица, 1 (Заельцовский район)",
|
||||
wantName: "медкадры", wantType: "улица",
|
||||
wantHouse: "1", wantHas: true,
|
||||
|
||||
@@ -2,73 +2,19 @@ package address
|
||||
|
||||
import "strings"
|
||||
|
||||
// fillerWords are tokens that carry no street-name meaning and that people
|
||||
// routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
|
||||
// Dropping them makes the service tolerant of natural phrasing. District
|
||||
// adjectives are deliberately NOT dropped: they can legitimately be part of a
|
||||
// street name.
|
||||
var fillerWords = map[string]bool{
|
||||
"новосибирск": true, "нск": true, "нсо": true,
|
||||
"город": true, "г": true,
|
||||
"область": true, "обл": true,
|
||||
"район": true, "дом": true, "д": true,
|
||||
}
|
||||
|
||||
// streetTypes maps a surface form (including common abbreviations) to a single
|
||||
// canonical street type. The type is informational for matching (we strip it
|
||||
// from the key) but useful for disambiguation and display.
|
||||
var streetTypes = map[string]string{
|
||||
"ул": "улица", "улица": "улица",
|
||||
"пер": "переулок", "пер-к": "переулок", "переулок": "переулок",
|
||||
"пр": "проезд", "проезд": "проезд",
|
||||
"пр-т": "проспект", "просп": "проспект", "проспект": "проспект",
|
||||
"ш": "шоссе", "шоссе": "шоссе",
|
||||
"мкр": "микрорайон", "микрорайон": "микрорайон",
|
||||
"туп": "тупик", "тупик": "тупик",
|
||||
"спуск": "спуск",
|
||||
"б-р": "бульвар", "бул": "бульвар", "бульвар": "бульвар",
|
||||
"пл": "площадь", "площадь": "площадь",
|
||||
"пос": "поселок", "поселок": "поселок",
|
||||
"тракт": "тракт",
|
||||
"аллея": "аллея",
|
||||
"наб": "набережная", "набережная": "набережная",
|
||||
"кольцо": "кольцо",
|
||||
"просека": "просека",
|
||||
"квартал": "квартал",
|
||||
"территория": "территория",
|
||||
"дорога": "дорога",
|
||||
"разъезд": "разъезд",
|
||||
}
|
||||
|
||||
// ordinalWords maps spelled-out ordinals to their digit form. Only the forms
|
||||
// that actually occur in practice are listed; unknown words are treated as
|
||||
// part of the street name.
|
||||
var ordinalWords = map[string]string{"первый": "1", "первая": "1", "первое": "1",
|
||||
"второй": "2", "вторая": "2", "второе": "2",
|
||||
"третий": "3", "третья": "3", "третье": "3",
|
||||
"четвертый": "4", "четвертая": "4", "четвертое": "4",
|
||||
"пятый": "5", "пятая": "5", "пятое": "5",
|
||||
"шестой": "6", "шестая": "6", "шестое": "6",
|
||||
"седьмой": "7", "седьмая": "7", "седьмое": "7",
|
||||
"восьмой": "8", "восьмая": "8", "восьмое": "8",
|
||||
"девятый": "9", "девятая": "9", "девятое": "9",
|
||||
"десятый": "10", "десятая": "10", "десятое": "10",
|
||||
}
|
||||
|
||||
// ordinalSuffixes are the gender/number endings that may follow an ordinal
|
||||
// digit when it is written as a separate token ("2 я", "10 й").
|
||||
var ordinalSuffixes = map[string]bool{
|
||||
"я": true, "й": true, "е": true,
|
||||
"ая": true, "ый": true, "ой": true, "ое": true,
|
||||
"ий": true, "ья": true, "ье": true, "ьи": true,
|
||||
}
|
||||
|
||||
// streetKey builds the canonical, type-independent street key:
|
||||
// "<ordinal>|<sorted name tokens>". The type word is dropped so that a user
|
||||
// who omits it ("станционная 1" instead of "Станционная улица 1") still
|
||||
// matches; type confusion is surfaced later as ambiguity.
|
||||
func streetKey(ordinal, nameKey string) string {
|
||||
return ordinal + "|" + nameKey
|
||||
// streetKey builds the canonical, city-aware, type-independent street key:
|
||||
//
|
||||
// "<cityKey>|<ordinal>|<sorted name tokens>"
|
||||
//
|
||||
// The city is part of the key so that "Ленина" in Novosibirsk and "Ленина" in
|
||||
// Berdsk are different streets. The type word is dropped so that a user who
|
||||
// omits it ("станционная 1" instead of "Станционная улица 1") still matches;
|
||||
// type confusion is surfaced later as ambiguity.
|
||||
//
|
||||
// None of the components may contain "|": NormalizeText folds all punctuation
|
||||
// that could appear in a name into spaces, and ordinals are digits only.
|
||||
func streetKey(cityKey, ordinal, nameKey string) string {
|
||||
return cityKey + "|" + ordinal + "|" + nameKey
|
||||
}
|
||||
|
||||
// normalizeOrdinal trims leading zeros so "02-я" and "2-я" agree.
|
||||
|
||||
@@ -10,7 +10,7 @@ package address
|
||||
// Status is the machine-readable outcome of an address resolution.
|
||||
//
|
||||
// It is intentionally explicit: callers must never have to guess whether an
|
||||
// empty result means "not found" or "the upstream failed".
|
||||
// empty result means "not found" or "the resolver refused to choose".
|
||||
type Status string
|
||||
|
||||
const (
|
||||
@@ -25,11 +25,6 @@ const (
|
||||
StatusAmbiguous Status = "ambiguous"
|
||||
// StatusNotFound means nothing in the registry matched.
|
||||
StatusNotFound Status = "not_found"
|
||||
// StatusPortalTimeout means the external outages portal did not answer in
|
||||
// time. It is an infrastructure outcome, not "no outages".
|
||||
StatusPortalTimeout Status = "portal_timeout"
|
||||
// StatusInfra is a generic terminal infrastructure failure.
|
||||
StatusInfra Status = "infra_error"
|
||||
)
|
||||
|
||||
// ParsedAddress is the structured form of a free-form address string.
|
||||
@@ -41,6 +36,13 @@ type ParsedAddress struct {
|
||||
Raw string `json:"raw"`
|
||||
Normalized string `json:"normalized"`
|
||||
|
||||
// City is the canonical city name when the query names one (or when the
|
||||
// parser is configured with a default city). CityKey is its normalized
|
||||
// form. The city is part of StreetKey, so identical street names in
|
||||
// different cities never collide.
|
||||
City string `json:"city,omitempty"`
|
||||
CityKey string `json:"city_key,omitempty"`
|
||||
|
||||
// Street components.
|
||||
//
|
||||
// NameKey is the order-insensitive canonical name (tokens sorted), so
|
||||
@@ -73,6 +75,9 @@ type Entry struct {
|
||||
Address string `json:"address"`
|
||||
District string `json:"district,omitempty"`
|
||||
|
||||
City string `json:"city,omitempty"`
|
||||
CityKey string `json:"city_key,omitempty"`
|
||||
|
||||
Ordinal string `json:"ordinal,omitempty"`
|
||||
NameKey string `json:"name_key,omitempty"`
|
||||
NameDisplay string `json:"name_display,omitempty"`
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
package match
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
)
|
||||
|
||||
// staticRewriter returns a fixed list of variants, emulating the LLM adapter.
|
||||
type staticRewriter []string
|
||||
|
||||
func (s staticRewriter) Rewrite(context.Context, string) ([]string, error) { return s, nil }
|
||||
|
||||
// TestRewriteInheritsDetectedCity guards the multi-city LLM path: the embedded
|
||||
// prompt tells the model to strip administrative words, so a correct variant
|
||||
// usually has no city. It must inherit the city detected in the original query,
|
||||
// while a variant naming a different city must still be rejected.
|
||||
func TestRewriteInheritsDetectedCity(t *testing.T) {
|
||||
parser := address.NewParser(address.NewDictionary(matchCityDictionary()), "")
|
||||
entries := cityEntries(t, parser,
|
||||
[2]string{"Новосибирск", "Кошурникова улица 5"},
|
||||
[2]string{"Бердск", "Кошурникова улица 5"},
|
||||
)
|
||||
// A threshold above any typo score forces the deterministic path to miss.
|
||||
ix := NewIndex("test", entries, Options{FuzzyThreshold: 0.999, MaxCandidates: 5})
|
||||
|
||||
cityless := NewResolverWithOptions(ix, staticRewriter{"Кошурникова 5"}, ResolverOptions{Parser: parser})
|
||||
res := cityless.ResolveInCity(context.Background(), "нск кошурникиова 5", "")
|
||||
if res.Status != address.StatusFound || res.City != "Новосибирск" {
|
||||
t.Fatalf("cityless variant: %+v", res)
|
||||
}
|
||||
|
||||
conflict := NewResolverWithOptions(ix, staticRewriter{"Бердск, Кошурникова 5"}, ResolverOptions{Parser: parser})
|
||||
res = conflict.ResolveInCity(context.Background(), "нск кошурникиова 5", "")
|
||||
if res.Status == address.StatusFound {
|
||||
t.Fatalf("conflicting city variant was accepted: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func matchCityDictionary() []address.City {
|
||||
return []address.City{
|
||||
{Name: "Новосибирск", Aliases: []string{"нск"}},
|
||||
{Name: "Бердск"},
|
||||
}
|
||||
}
|
||||
|
||||
func cityEntries(t *testing.T, parser *address.Parser, rows ...[2]string) []address.Entry {
|
||||
t.Helper()
|
||||
entries := make([]address.Entry, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
p := parser.ParseInCity(row[1], row[0])
|
||||
entries = append(entries, address.Entry{
|
||||
ID: row[0] + "/" + row[1], Address: row[1],
|
||||
City: p.City, CityKey: p.CityKey,
|
||||
Ordinal: p.Ordinal, NameKey: p.NameKey, NameDisplay: p.NameDisplay,
|
||||
StreetType: p.StreetType, StreetKey: p.StreetKey,
|
||||
House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter,
|
||||
Building: p.Building, BuildingFrac: p.BuildingFrac, BuildingLetter: p.BuildingLetter,
|
||||
Structure: p.Structure,
|
||||
})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func TestIndexKeepsCitiesApart(t *testing.T) {
|
||||
parser := address.NewParser(address.NewDictionary(matchCityDictionary()), "")
|
||||
entries := cityEntries(t, parser,
|
||||
[2]string{"Новосибирск", "Ленина улица 5"},
|
||||
[2]string{"Бердск", "Ленина улица 5"},
|
||||
)
|
||||
resolver := NewResolverWithOptions(
|
||||
NewIndex("test", entries, DefaultOptions()),
|
||||
nil,
|
||||
ResolverOptions{Parser: parser},
|
||||
)
|
||||
|
||||
nsk := resolver.ResolveInCity(t.Context(), "Ленина 5", "Новосибирск")
|
||||
if nsk.Status != address.StatusFound || nsk.City != "Новосибирск" {
|
||||
t.Fatalf("Новосибирск: %+v", nsk)
|
||||
}
|
||||
berdsk := resolver.ResolveInCity(t.Context(), "Ленина 5", "Бердск")
|
||||
if berdsk.Status != address.StatusFound || berdsk.City != "Бердск" {
|
||||
t.Fatalf("Бердск: %+v", berdsk)
|
||||
}
|
||||
|
||||
// Without a city the resolver must not pick one: both streets are exact
|
||||
// matches in different cities.
|
||||
res := resolver.Resolve(t.Context(), "Ленина 5")
|
||||
if res.Status != address.StatusAmbiguous {
|
||||
t.Fatalf("cityless status = %q, want ambiguous (candidates: %+v)", res.Status, res.Candidates)
|
||||
}
|
||||
if len(res.Candidates) != 2 {
|
||||
t.Fatalf("candidates = %+v, want one per city", res.Candidates)
|
||||
}
|
||||
if res.City != "" {
|
||||
t.Fatalf("ambiguous cityless result must not report a city, got %q", res.City)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCitylessQueryReportsTheOnlyCity(t *testing.T) {
|
||||
parser := address.NewParser(address.NewDictionary(matchCityDictionary()), "")
|
||||
entries := cityEntries(t, parser, [2]string{"Новосибирск", "Ленина улица 5"})
|
||||
resolver := NewResolverWithOptions(
|
||||
NewIndex("test", entries, DefaultOptions()),
|
||||
nil,
|
||||
ResolverOptions{Parser: parser},
|
||||
)
|
||||
|
||||
res := resolver.Resolve(t.Context(), "Ленина 5")
|
||||
if res.Status != address.StatusFound || res.City != "Новосибирск" {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructureIsPartOfHouseIdentity(t *testing.T) {
|
||||
ix := buildIndex(t,
|
||||
"Станционная улица 1",
|
||||
"Станционная улица 1 стр. 2",
|
||||
)
|
||||
|
||||
found := ix.Resolve(address.Parse("станционная 1 стр. 2"))
|
||||
if found.Status != address.StatusFound || found.Address != "Станционная улица 1 стр. 2" {
|
||||
t.Fatalf("exact structure: %+v", found)
|
||||
}
|
||||
|
||||
// A structure that does not exist must never resolve to another building.
|
||||
wrong := ix.Resolve(address.Parse("станционная 1 стр. 3"))
|
||||
if wrong.Status == address.StatusFound {
|
||||
t.Fatalf("wrong structure resolved to %q", wrong.Address)
|
||||
}
|
||||
if wrong.Status != address.StatusAmbiguous {
|
||||
t.Fatalf("wrong structure status = %q, want ambiguous", wrong.Status)
|
||||
}
|
||||
|
||||
base := ix.Resolve(address.Parse("станционная 1"))
|
||||
if base.Status != address.StatusFound || base.Address != "Станционная улица 1" {
|
||||
t.Fatalf("base house: %+v", base)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexOrderIsDeterministic(t *testing.T) {
|
||||
addresses := []string{
|
||||
"Станционная улица 1/1",
|
||||
"Станционная улица 2/1",
|
||||
"Станционная улица 3/1 к1",
|
||||
"Станционная улица 3/1 к2",
|
||||
"Искитимская улица 5",
|
||||
}
|
||||
forward := buildEntries(t, addresses...)
|
||||
backward := buildEntries(t, reverse(addresses)...)
|
||||
|
||||
a := NewIndex("v", forward, DefaultOptions())
|
||||
b := NewIndex("v", backward, DefaultOptions())
|
||||
|
||||
for _, q := range []string{"станционная 3/1", "станционная 1/2", "искитимсая 5"} {
|
||||
ra := a.Resolve(address.Parse(q))
|
||||
rb := b.Resolve(address.Parse(q))
|
||||
if !reflect.DeepEqual(ra, rb) {
|
||||
t.Fatalf("%q: results differ\n a=%+v\n b=%+v", q, ra, rb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongTokenDoesNotMatch(t *testing.T) {
|
||||
ix := buildIndex(t, "Станционная улица 1")
|
||||
res := ix.Resolve(address.Parse(strings.Repeat("а", 4000)))
|
||||
if res.Status != address.StatusNotFound {
|
||||
t.Fatalf("status = %q, want not_found", res.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func buildEntries(t *testing.T, addresses ...string) []address.Entry {
|
||||
t.Helper()
|
||||
entries := make([]address.Entry, 0, len(addresses))
|
||||
for _, a := range addresses {
|
||||
p := address.Parse(a)
|
||||
entries = append(entries, address.Entry{
|
||||
ID: a, Address: a,
|
||||
Ordinal: p.Ordinal, NameKey: p.NameKey, NameDisplay: p.NameDisplay,
|
||||
StreetType: p.StreetType, StreetKey: p.StreetKey,
|
||||
House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter,
|
||||
Building: p.Building, BuildingFrac: p.BuildingFrac, BuildingLetter: p.BuildingLetter,
|
||||
Structure: p.Structure,
|
||||
})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func reverse(in []string) []string {
|
||||
out := make([]string, len(in))
|
||||
for i, s := range in {
|
||||
out[len(in)-1-i] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
+137
-32
@@ -9,6 +9,7 @@ package match
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
)
|
||||
@@ -16,16 +17,19 @@ import (
|
||||
// Candidate is one plausible match, returned alongside a non-"found" result so
|
||||
// the caller (or an operator) can see what the resolver was choosing between.
|
||||
type Candidate struct {
|
||||
Address string `json:"address"`
|
||||
District string `json:"district,omitempty"`
|
||||
House string `json:"house,omitempty"`
|
||||
Score float64 `json:"score"`
|
||||
Address string `json:"address"`
|
||||
City string `json:"city,omitempty"`
|
||||
District string `json:"district,omitempty"`
|
||||
House string `json:"house,omitempty"`
|
||||
Structure string `json:"structure,omitempty"`
|
||||
Score float64 `json:"score"`
|
||||
}
|
||||
|
||||
// Result is the outcome of a resolution.
|
||||
type Result struct {
|
||||
Status address.Status `json:"status"`
|
||||
Address string `json:"address,omitempty"`
|
||||
City string `json:"city,omitempty"`
|
||||
District string `json:"district,omitempty"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Candidates []Candidate `json:"candidates,omitempty"`
|
||||
@@ -59,10 +63,10 @@ func DefaultOptions() Options {
|
||||
}
|
||||
|
||||
type streetRef struct {
|
||||
key string
|
||||
ordinal string
|
||||
nameKey string
|
||||
streetType string
|
||||
key string
|
||||
cityKey string
|
||||
ordinal string
|
||||
nameKey string
|
||||
}
|
||||
|
||||
// Index is an immutable, in-memory registry index. It is safe for concurrent
|
||||
@@ -72,26 +76,46 @@ type Index struct {
|
||||
opts Options
|
||||
byKey map[string][]address.Entry
|
||||
streets []streetRef
|
||||
cities map[string]string // cityKey -> canonical display name
|
||||
size int
|
||||
}
|
||||
|
||||
// maxFuzzyNameRunes bounds the fuzzy scan. No real street name is longer than
|
||||
// this; without the bound a single huge token would make every Levenshtein
|
||||
// comparison walk a very long string and turn a request into a CPU sink.
|
||||
const maxFuzzyNameRunes = 96
|
||||
|
||||
// NewIndex builds an index from canonical entries. Entries sharing a StreetKey
|
||||
// are grouped; distinct streets are collected for typo-tolerant matching.
|
||||
//
|
||||
// Entries are sorted first, so candidate order is deterministic regardless of
|
||||
// the order in which the storage adapter returned them.
|
||||
func NewIndex(version string, entries []address.Entry, opts Options) *Index {
|
||||
sorted := append([]address.Entry(nil), entries...)
|
||||
sort.SliceStable(sorted, func(i, j int) bool { return entryLess(sorted[i], sorted[j]) })
|
||||
|
||||
ix := &Index{
|
||||
version: version,
|
||||
opts: opts,
|
||||
byKey: make(map[string][]address.Entry, len(entries)),
|
||||
byKey: make(map[string][]address.Entry, len(sorted)),
|
||||
cities: make(map[string]string),
|
||||
size: len(sorted),
|
||||
}
|
||||
seen := make(map[string]struct{}, len(entries))
|
||||
for _, e := range entries {
|
||||
seen := make(map[string]struct{}, len(sorted))
|
||||
for _, e := range sorted {
|
||||
ix.byKey[e.StreetKey] = append(ix.byKey[e.StreetKey], e)
|
||||
if e.CityKey != "" {
|
||||
if _, ok := ix.cities[e.CityKey]; !ok {
|
||||
ix.cities[e.CityKey] = e.City
|
||||
}
|
||||
}
|
||||
if _, ok := seen[e.StreetKey]; !ok {
|
||||
seen[e.StreetKey] = struct{}{}
|
||||
ix.streets = append(ix.streets, streetRef{
|
||||
key: e.StreetKey,
|
||||
ordinal: e.Ordinal,
|
||||
nameKey: e.NameKey,
|
||||
streetType: e.StreetType,
|
||||
key: e.StreetKey,
|
||||
cityKey: e.CityKey,
|
||||
ordinal: e.Ordinal,
|
||||
nameKey: e.NameKey,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -99,22 +123,57 @@ func NewIndex(version string, entries []address.Entry, opts Options) *Index {
|
||||
return ix
|
||||
}
|
||||
|
||||
// entryLess defines the canonical order of registry rows. Address strings are
|
||||
// the last meaningful tie-breaker; ID makes the order total even for duplicated
|
||||
// rows.
|
||||
func entryLess(a, b address.Entry) bool {
|
||||
switch {
|
||||
case a.CityKey != b.CityKey:
|
||||
return a.CityKey < b.CityKey
|
||||
case a.StreetKey != b.StreetKey:
|
||||
return a.StreetKey < b.StreetKey
|
||||
case a.House != b.House:
|
||||
return a.House < b.House
|
||||
case a.HouseFrac != b.HouseFrac:
|
||||
return a.HouseFrac < b.HouseFrac
|
||||
case a.HouseLetter != b.HouseLetter:
|
||||
return a.HouseLetter < b.HouseLetter
|
||||
case a.Building != b.Building:
|
||||
return a.Building < b.Building
|
||||
case a.BuildingFrac != b.BuildingFrac:
|
||||
return a.BuildingFrac < b.BuildingFrac
|
||||
case a.BuildingLetter != b.BuildingLetter:
|
||||
return a.BuildingLetter < b.BuildingLetter
|
||||
case a.Structure != b.Structure:
|
||||
return a.Structure < b.Structure
|
||||
case a.Address != b.Address:
|
||||
return a.Address < b.Address
|
||||
default:
|
||||
return a.ID < b.ID
|
||||
}
|
||||
}
|
||||
|
||||
// Version returns the data version the index was built from.
|
||||
func (ix *Index) Version() string { return ix.version }
|
||||
|
||||
// Size returns the number of indexed entries.
|
||||
func (ix *Index) Size() int {
|
||||
n := 0
|
||||
for _, v := range ix.byKey {
|
||||
n += len(v)
|
||||
func (ix *Index) Size() int { return ix.size }
|
||||
|
||||
// Cities returns the distinct canonical city names present in the index, in
|
||||
// deterministic order. A city-less index returns an empty slice.
|
||||
func (ix *Index) Cities() []string {
|
||||
out := make([]string, 0, len(ix.cities))
|
||||
for _, name := range ix.cities {
|
||||
out = append(out, name)
|
||||
}
|
||||
return n
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Resolve resolves a parsed address. It never returns an address that is not
|
||||
// present in the index.
|
||||
func (ix *Index) Resolve(p address.ParsedAddress) Result {
|
||||
res := Result{DataVersion: ix.version}
|
||||
res := Result{DataVersion: ix.version, City: p.City}
|
||||
|
||||
entries := ix.byKey[p.StreetKey]
|
||||
similarity := 1.0
|
||||
@@ -133,6 +192,11 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
|
||||
res.Status = address.StatusNotFound
|
||||
return res
|
||||
}
|
||||
if res.City == "" {
|
||||
// A city-less query may still match exactly one city; report it so the
|
||||
// caller does not have to re-derive the city from the candidates.
|
||||
res.City = commonCity(entries)
|
||||
}
|
||||
|
||||
if !p.HasHouse {
|
||||
// Street is known but the caller did not supply a house: safe but
|
||||
@@ -148,6 +212,7 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
|
||||
case 1:
|
||||
res.Status = address.StatusFound
|
||||
res.Address = matched[0].Address
|
||||
res.City = matched[0].City
|
||||
res.District = matched[0].District
|
||||
res.Confidence = clamp01(similarity)
|
||||
return res
|
||||
@@ -162,7 +227,14 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
|
||||
}
|
||||
res.Status = address.StatusPartial
|
||||
res.Confidence = clamp01(similarity * 0.9)
|
||||
res.Candidates = candidates(entries, similarity, ix.opts.MaxCandidates)
|
||||
// Prefer houses that share the number/fraction/letter: they are the
|
||||
// plausible corrections (a different corpus or structure), while the
|
||||
// rest of the street is noise.
|
||||
related := matchHouseUnitAll(entries, p)
|
||||
if len(related) == 0 {
|
||||
related = entries
|
||||
}
|
||||
res.Candidates = candidates(related, similarity, ix.opts.MaxCandidates)
|
||||
return res
|
||||
default:
|
||||
res.Status = address.StatusAmbiguous
|
||||
@@ -176,12 +248,16 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
|
||||
// the best street when the match is unambiguous, or the list of competing
|
||||
// street keys when several are within the ambiguity margin.
|
||||
func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry, similarity float64, ambiguousKeys []string) {
|
||||
if p.NameKey == "" {
|
||||
if p.NameKey == "" || utf8.RuneCountInString(p.NameKey) > maxFuzzyNameRunes {
|
||||
return nil, 0, nil
|
||||
}
|
||||
best := 0.0
|
||||
bestKey := ""
|
||||
for _, s := range ix.streets {
|
||||
// A query that names a city may only match streets in that city.
|
||||
if p.CityKey != "" && s.cityKey != p.CityKey {
|
||||
continue
|
||||
}
|
||||
// If the query carries an ordinal, the candidate's ordinal must match:
|
||||
// a user who says "1-й Северный" does not mean the plain "Северная".
|
||||
if !ordinalCompatible(p.Ordinal, s.ordinal) {
|
||||
@@ -197,6 +273,9 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
|
||||
}
|
||||
|
||||
for _, s := range ix.streets {
|
||||
if p.CityKey != "" && s.cityKey != p.CityKey {
|
||||
continue
|
||||
}
|
||||
if !ordinalCompatible(p.Ordinal, s.ordinal) {
|
||||
continue
|
||||
}
|
||||
@@ -232,10 +311,12 @@ func (ix *Index) representatives(keys []string, score float64) []Candidate {
|
||||
}
|
||||
e := es[0]
|
||||
out = append(out, Candidate{
|
||||
Address: e.Address,
|
||||
District: e.District,
|
||||
House: e.House,
|
||||
Score: clamp01(score),
|
||||
Address: e.Address,
|
||||
City: e.City,
|
||||
District: e.District,
|
||||
House: e.House,
|
||||
Structure: e.Structure,
|
||||
Score: clamp01(score),
|
||||
})
|
||||
if len(out) == ix.opts.MaxCandidates {
|
||||
break
|
||||
@@ -275,13 +356,16 @@ func sameHouseSeveral(entries []address.Entry, p address.ParsedAddress) bool {
|
||||
|
||||
// houseUnitEqual compares the parts of a house unit. Unspecified (empty) parts
|
||||
// must also be empty on the entry side, so "60/1" never matches "60/1 к148".
|
||||
// The structure ("стр. 2") is part of the identity: a caller who explicitly
|
||||
// asked for a different structure must not be handed another building.
|
||||
func houseUnitEqual(e address.Entry, p address.ParsedAddress) bool {
|
||||
return e.House == p.House &&
|
||||
e.HouseFrac == p.HouseFrac &&
|
||||
e.HouseLetter == p.HouseLetter &&
|
||||
e.Building == p.Building &&
|
||||
e.BuildingFrac == p.BuildingFrac &&
|
||||
e.BuildingLetter == p.BuildingLetter
|
||||
e.BuildingLetter == p.BuildingLetter &&
|
||||
e.Structure == p.Structure
|
||||
}
|
||||
|
||||
func candidates(entries []address.Entry, score float64, limit int) []Candidate {
|
||||
@@ -291,10 +375,12 @@ func candidates(entries []address.Entry, score float64, limit int) []Candidate {
|
||||
out := make([]Candidate, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
out = append(out, Candidate{
|
||||
Address: e.Address,
|
||||
District: e.District,
|
||||
House: e.House,
|
||||
Score: clamp01(score),
|
||||
Address: e.Address,
|
||||
City: e.City,
|
||||
District: e.District,
|
||||
House: e.House,
|
||||
Structure: e.Structure,
|
||||
Score: clamp01(score),
|
||||
})
|
||||
if len(out) == limit {
|
||||
break
|
||||
@@ -312,3 +398,22 @@ func clamp01(v float64) float64 {
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// commonCity returns the city shared by all entries, or "" when they span
|
||||
// cities (or carry no city at all).
|
||||
func commonCity(entries []address.Entry) string {
|
||||
key, name := "", ""
|
||||
for _, e := range entries {
|
||||
if e.CityKey == "" {
|
||||
continue
|
||||
}
|
||||
if key == "" {
|
||||
key, name = e.CityKey, e.City
|
||||
continue
|
||||
}
|
||||
if e.CityKey != key {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package match
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// levenshtein returns the edit distance between two strings, measured in
|
||||
// runes. It uses the rolling-row formulation: O(min(n,m)) memory.
|
||||
@@ -102,18 +105,30 @@ func min3(a, b, c int) int {
|
||||
return m
|
||||
}
|
||||
|
||||
// maxTokenRunes is the longest token the fuzzy matcher will compare. Real
|
||||
// street-name tokens are far shorter; anything longer is noise or abuse.
|
||||
const maxTokenRunes = 64
|
||||
|
||||
// tokenScore scores two name tokens. A single edit is tolerated for tokens of
|
||||
// length >= 4 and two edits for length >= 8; otherwise it falls back to the
|
||||
// plain ratio. Short names (<= 3 chars) are intentionally not fuzzy-matched:
|
||||
// one edit is too large a fraction of the word to be safe.
|
||||
//
|
||||
// Tokens longer than maxTokenRunes are never similar to a real street name.
|
||||
// Rejecting them up front keeps a pathological input from turning one request
|
||||
// into millions of Levenshtein cell updates.
|
||||
func tokenScore(a, b string) float64 {
|
||||
if a == b {
|
||||
return 1
|
||||
}
|
||||
ra, rb := utf8.RuneCountInString(a), utf8.RuneCountInString(b)
|
||||
if ra == 0 || rb == 0 || ra > maxTokenRunes || rb > maxTokenRunes {
|
||||
return 0
|
||||
}
|
||||
d := levenshtein(a, b)
|
||||
minLen := len([]rune(a))
|
||||
if lb := len([]rune(b)); lb < minLen {
|
||||
minLen = lb
|
||||
minLen := ra
|
||||
if rb < minLen {
|
||||
minLen = rb
|
||||
}
|
||||
switch {
|
||||
case d == 1 && minLen >= 4:
|
||||
|
||||
@@ -20,6 +20,25 @@ type Rewriter interface {
|
||||
Rewrite(ctx context.Context, raw string) ([]string, error)
|
||||
}
|
||||
|
||||
// Parser is the address parser used by the resolver. It is defined here
|
||||
// (consumer side) so the resolver can be tested with a stub and so a
|
||||
// deployment can plug in its own dictionary.
|
||||
type Parser interface {
|
||||
Parse(raw string) address.ParsedAddress
|
||||
ParseInCity(raw, city string) address.ParsedAddress
|
||||
MatchCity(query string) (address.City, bool)
|
||||
}
|
||||
|
||||
// ResolverOptions tunes optional resolver behaviour.
|
||||
type ResolverOptions struct {
|
||||
// Parser defaults to address.DefaultParser().
|
||||
Parser Parser
|
||||
// OnRewriteError, when set, observes failures of the optional rewriter.
|
||||
// The resolver always keeps the deterministic result, but operators still
|
||||
// need to know that the fallback is broken.
|
||||
OnRewriteError func(error)
|
||||
}
|
||||
|
||||
// Resolver wraps an Index with an optional Rewriter.
|
||||
//
|
||||
// The deterministic path is always tried first. The rewriter is consulted only
|
||||
@@ -27,13 +46,30 @@ type Rewriter interface {
|
||||
// suggestions must pass compatibleVariant before being considered, so the
|
||||
// model cannot substitute a different street or corrupt the house unit.
|
||||
type Resolver struct {
|
||||
index *Index
|
||||
rewriter Rewriter
|
||||
index *Index
|
||||
rewriter Rewriter
|
||||
parser Parser
|
||||
onRewriteError func(error)
|
||||
}
|
||||
|
||||
// NewResolver builds a resolver. rewriter may be nil to disable rewriting.
|
||||
// NewResolver builds a resolver with the default parser and no rewriter
|
||||
// diagnostics.
|
||||
func NewResolver(index *Index, rewriter Rewriter) *Resolver {
|
||||
return &Resolver{index: index, rewriter: rewriter}
|
||||
return NewResolverWithOptions(index, rewriter, ResolverOptions{})
|
||||
}
|
||||
|
||||
// NewResolverWithOptions builds a resolver with explicit options.
|
||||
func NewResolverWithOptions(index *Index, rewriter Rewriter, opts ResolverOptions) *Resolver {
|
||||
parser := opts.Parser
|
||||
if parser == nil {
|
||||
parser = address.DefaultParser()
|
||||
}
|
||||
return &Resolver{
|
||||
index: index,
|
||||
rewriter: rewriter,
|
||||
parser: parser,
|
||||
onRewriteError: opts.OnRewriteError,
|
||||
}
|
||||
}
|
||||
|
||||
// Version returns the current data version.
|
||||
@@ -42,9 +78,35 @@ func (r *Resolver) Version() string { return r.index.Version() }
|
||||
// Ready reports whether the resolver has a usable index.
|
||||
func (r *Resolver) Ready() bool { return r.index != nil && r.index.Size() > 0 }
|
||||
|
||||
// Cities returns the canonical city names present in the index.
|
||||
func (r *Resolver) Cities() []string {
|
||||
if r.index == nil {
|
||||
return nil
|
||||
}
|
||||
return r.index.Cities()
|
||||
}
|
||||
|
||||
// MatchCity reports whether query is a city name/alias and returns the
|
||||
// canonical name. It uses the resolver's parser dictionary, not the index, so
|
||||
// it also recognizes aliases for cities that have no rows yet.
|
||||
func (r *Resolver) MatchCity(query string) (string, bool) {
|
||||
city, ok := r.parser.MatchCity(query)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return city.Name, true
|
||||
}
|
||||
|
||||
// Resolve parses raw and resolves it, consulting the rewriter when enabled.
|
||||
func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
|
||||
parsed := address.Parse(raw)
|
||||
return r.ResolveInCity(ctx, raw, "")
|
||||
}
|
||||
|
||||
// ResolveInCity is Resolve with an explicit city hint. The hint overrides a
|
||||
// city found in the address text and lets clients pass the city as a separate
|
||||
// field.
|
||||
func (r *Resolver) ResolveInCity(ctx context.Context, raw, city string) Result {
|
||||
parsed := r.parse(raw, city)
|
||||
res := r.index.Resolve(parsed)
|
||||
|
||||
if res.Status == address.StatusFound || r.rewriter == nil {
|
||||
@@ -55,13 +117,27 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
|
||||
}
|
||||
|
||||
alternatives, err := r.rewriter.Rewrite(ctx, raw)
|
||||
if err != nil || len(alternatives) == 0 {
|
||||
if err != nil {
|
||||
if r.onRewriteError != nil {
|
||||
r.onRewriteError(err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
if len(alternatives) == 0 {
|
||||
return res
|
||||
}
|
||||
|
||||
best := res
|
||||
for _, alt := range alternatives {
|
||||
variant := address.Parse(alt)
|
||||
variant := r.parse(alt, city)
|
||||
// The embedded prompt asks the model to strip administrative words
|
||||
// (including the city), so a correct variant often has no city. It
|
||||
// inherits the original's city; only a conflicting explicit city is a
|
||||
// disqualification. Without this, the rewriter silently contributes
|
||||
// nothing in multi-city deployments (where default_city is empty).
|
||||
if variant.CityKey == "" && parsed.CityKey != "" {
|
||||
variant = r.parser.ParseInCity(alt, parsed.City)
|
||||
}
|
||||
// The model may fix the spelling of the street, nothing else.
|
||||
if !compatibleVariant(parsed, variant) {
|
||||
continue
|
||||
@@ -74,6 +150,13 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
|
||||
return best
|
||||
}
|
||||
|
||||
func (r *Resolver) parse(raw, city string) address.ParsedAddress {
|
||||
if strings.TrimSpace(city) != "" {
|
||||
return r.parser.ParseInCity(raw, city)
|
||||
}
|
||||
return r.parser.Parse(raw)
|
||||
}
|
||||
|
||||
// compatibleVariant reports whether an LLM rewrite may be considered. A rewrite
|
||||
// is acceptable only when it:
|
||||
// - keeps the exact house unit (number, fraction, letter, corpus, structure
|
||||
@@ -82,6 +165,9 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
|
||||
// - has the same words in the street name, each within a small edit distance
|
||||
// — so a different street can never be substituted.
|
||||
func compatibleVariant(orig, variant address.ParsedAddress) bool {
|
||||
if orig.CityKey != variant.CityKey {
|
||||
return false
|
||||
}
|
||||
if !sameHouseUnit(orig, variant) {
|
||||
return false
|
||||
}
|
||||
@@ -108,9 +194,13 @@ func nearToken(a, b string) bool {
|
||||
if a == b {
|
||||
return true
|
||||
}
|
||||
minLen := len([]rune(a))
|
||||
if lb := len([]rune(b)); lb < minLen {
|
||||
minLen = lb
|
||||
ra, rb := len([]rune(a)), len([]rune(b))
|
||||
if ra == 0 || rb == 0 || ra > maxTokenRunes || rb > maxTokenRunes {
|
||||
return false
|
||||
}
|
||||
minLen := ra
|
||||
if rb < minLen {
|
||||
minLen = rb
|
||||
}
|
||||
d := damerau(a, b)
|
||||
switch {
|
||||
@@ -129,7 +219,8 @@ func sameHouseUnit(a, b address.ParsedAddress) bool {
|
||||
a.HouseLetter == b.HouseLetter &&
|
||||
a.Building == b.Building &&
|
||||
a.BuildingFrac == b.BuildingFrac &&
|
||||
a.BuildingLetter == b.BuildingLetter
|
||||
a.BuildingLetter == b.BuildingLetter &&
|
||||
a.Structure == b.Structure
|
||||
}
|
||||
|
||||
// better ranks results: a stronger status wins, then higher confidence.
|
||||
|
||||
@@ -42,6 +42,26 @@ func TestCompatibleVariantProtectsApartment(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompatibleVariantProtectsCityAndStructure makes sure a rewrite cannot
|
||||
// move the address to another city or another building.
|
||||
func TestCompatibleVariantProtectsCityAndStructure(t *testing.T) {
|
||||
parser := address.NewParser(address.NewDictionary([]address.City{
|
||||
{Name: "Новосибирск", Aliases: []string{"нск"}},
|
||||
{Name: "Бердск"},
|
||||
}), "")
|
||||
orig := parser.Parse("нск, Сухарная улица 95 стр. 1")
|
||||
|
||||
if !compatibleVariant(orig, parser.Parse("Новосибирск, Сухраная улица 95 стр. 1")) {
|
||||
t.Error("same-city spelling fix must be accepted")
|
||||
}
|
||||
if compatibleVariant(orig, parser.Parse("Бердск, Сухарная улица 95 стр. 1")) {
|
||||
t.Error("city change must be rejected")
|
||||
}
|
||||
if compatibleVariant(orig, parser.Parse("нск, Сухарная улица 95 стр. 2")) {
|
||||
t.Error("structure change must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDamerau(t *testing.T) {
|
||||
if got := damerau("гоглоя", "гоголя"); got != 1 {
|
||||
t.Errorf("transposition distance = %d, want 1", got)
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
// Package outages contains the deterministic, pure logic for selecting outages
|
||||
// that belong to a concrete house unit. It does not talk to the network: the
|
||||
// portal client lives in an adapter and feeds this package.
|
||||
package outages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
)
|
||||
|
||||
// Outage is a single outage record as returned by the external portal.
|
||||
type Outage struct {
|
||||
Type string `json:"type,omitempty"`
|
||||
System string `json:"system,omitempty"`
|
||||
Address string `json:"address,omitempty"`
|
||||
StartDate string `json:"start_date,omitempty"`
|
||||
EndDate string `json:"end_date,omitempty"`
|
||||
Time string `json:"time,omitempty"`
|
||||
Action string `json:"action,omitempty"`
|
||||
|
||||
// Unit is the structured house unit, filled by the source adapter so the
|
||||
// domain never has to parse the raw external address format. Not exported
|
||||
// to JSON: it is an internal matching key.
|
||||
Unit HouseUnit `json:"-"`
|
||||
}
|
||||
|
||||
// HouseUnit identifies a concrete house within a street: number, optional
|
||||
// fraction and letter, and the corpus (building) with its own fraction and
|
||||
// letter. "Outages by house" matches on exactly this.
|
||||
type HouseUnit struct {
|
||||
House string
|
||||
HouseFrac string
|
||||
HouseLetter string
|
||||
Building string
|
||||
BuildingFrac string
|
||||
BuildingLetter string
|
||||
}
|
||||
|
||||
// HouseUnitOf extracts the unit from a parsed address.
|
||||
func HouseUnitOf(p address.ParsedAddress) HouseUnit {
|
||||
return HouseUnit{
|
||||
House: p.House,
|
||||
HouseFrac: p.HouseFrac,
|
||||
HouseLetter: p.HouseLetter,
|
||||
Building: p.Building,
|
||||
BuildingFrac: p.BuildingFrac,
|
||||
BuildingLetter: p.BuildingLetter,
|
||||
}
|
||||
}
|
||||
|
||||
// Source is anything that can answer "what outages are there for this
|
||||
// canonical address". It is implemented by the ArcGIS portal adapter and by
|
||||
// fakes in tests.
|
||||
type Source interface {
|
||||
Query(ctx context.Context, canonical string) ([]Outage, error)
|
||||
}
|
||||
|
||||
// MatchingHouse keeps only the outages that address the same house unit as the
|
||||
// target. This is the fix for the classic bug where a substring match returned
|
||||
// a neighbouring house's outage: matching is done on the parsed structure, not
|
||||
// on the raw text.
|
||||
func MatchingHouse(list []Outage, target address.ParsedAddress) []Outage {
|
||||
want := HouseUnitOf(target)
|
||||
out := make([]Outage, 0, len(list))
|
||||
for _, o := range list {
|
||||
unit := o.Unit
|
||||
if unit == (HouseUnit{}) {
|
||||
// The source did not pre-parse the address (older/alternative
|
||||
// adapters); fall back to parsing it here. Adapters should set
|
||||
// Unit so the domain stays independent of external formats.
|
||||
unit = HouseUnitOf(address.Parse(o.Address))
|
||||
}
|
||||
if unit == want {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Current keeps outages whose [start, end] range covers now. The portal
|
||||
// already filters by date, but we re-check locally because the portal's notion
|
||||
// of "today" and ours must agree (Asia/Novosibirsk, UTC+7).
|
||||
func Current(list []Outage, now time.Time, loc *time.Location) []Outage {
|
||||
out := make([]Outage, 0, len(list))
|
||||
for _, o := range list {
|
||||
start, okStart := parsePortalTime(o.StartDate, loc)
|
||||
end, okEnd := parsePortalTime(o.EndDate, loc)
|
||||
if !okStart || !okEnd {
|
||||
// Undated records are kept: better to report a possibly-current
|
||||
// outage than to silently drop it.
|
||||
out = append(out, o)
|
||||
continue
|
||||
}
|
||||
if (now.Equal(start) || now.After(start)) && (now.Equal(end) || now.Before(end)) {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parsePortalTime accepts the shapes the portal may produce: epoch
|
||||
// milliseconds, epoch seconds, RFC3339 and a few common date formats. All
|
||||
// results are interpreted in loc.
|
||||
func parsePortalTime(s string, loc *time.Location) (time.Time, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return time.Time{}, false
|
||||
}
|
||||
if n, err := strconv.ParseInt(s, 10, 64); err == nil {
|
||||
// Heuristic: 13+ digits is milliseconds, otherwise seconds.
|
||||
if n > 1e12 {
|
||||
return time.UnixMilli(n).In(loc), true
|
||||
}
|
||||
return time.Unix(n, 0).In(loc), true
|
||||
}
|
||||
layouts := []string{
|
||||
time.RFC3339,
|
||||
"2006-01-02T15:04:05",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02",
|
||||
}
|
||||
for _, layout := range layouts {
|
||||
if t, err := time.ParseInLocation(layout, s, loc); err == nil {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package outages
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
)
|
||||
|
||||
func TestMatchingHouse(t *testing.T) {
|
||||
list := []Outage{
|
||||
{Address: "Станционная улица 1/1", System: "Теплоснабжение"},
|
||||
{Address: "Станционная улица 1/2", System: "Теплоснабжение"},
|
||||
{Address: "Станционная улица 1/1 к1", System: "Теплоснабжение"},
|
||||
{Address: "Станционная улица 10", System: "Теплоснабжение"},
|
||||
}
|
||||
target := address.Parse("2-я Станционная улица 1/1")
|
||||
|
||||
got := MatchingHouse(list, target)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("expected exactly 1 matching outage, got %d: %+v", len(got), got)
|
||||
}
|
||||
if got[0].Address != "Станционная улица 1/1" {
|
||||
t.Errorf("matched wrong outage: %q", got[0].Address)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchingHouseIgnoresStructure(t *testing.T) {
|
||||
list := []Outage{{Address: "Станционная улица 1/1 стр.2"}}
|
||||
target := address.Parse("станционная 1/1 стр.3")
|
||||
if got := MatchingHouse(list, target); len(got) != 1 {
|
||||
t.Fatalf("structure must not affect house matching; got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// TestMatchingHouseDistrictSuffix covers the fallback path (Unit not set by the
|
||||
// adapter): the parser must itself drop the portal's "(<district> район)".
|
||||
func TestMatchingHouseDistrictSuffix(t *testing.T) {
|
||||
list := []Outage{{Address: "Медкадры улица, 1 (Заельцовский район)"}}
|
||||
target := address.Parse("Медкадры улица 1")
|
||||
if got := MatchingHouse(list, target); len(got) != 1 {
|
||||
t.Fatalf("district suffix must not break house matching; got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// TestMatchingHouseUsesStructuredUnit verifies the domain matches on the unit
|
||||
// the adapter parsed, not on the raw address text.
|
||||
func TestMatchingHouseUsesStructuredUnit(t *testing.T) {
|
||||
list := []Outage{{Address: "неважно", Unit: HouseUnit{House: "5", HouseFrac: "1"}}}
|
||||
if got := MatchingHouse(list, address.Parse("что угодно 5/1")); len(got) != 1 {
|
||||
t.Fatalf("structured unit not used for matching; got %d", len(got))
|
||||
}
|
||||
if got := MatchingHouse(list, address.Parse("что угодно 5/2")); len(got) != 0 {
|
||||
t.Fatalf("different unit must not match; got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurrent(t *testing.T) {
|
||||
loc := time.FixedZone("NSK", 7*3600)
|
||||
now := time.Date(2026, 10, 9, 12, 0, 0, 0, loc)
|
||||
within := now.Add(-time.Hour).UnixMilli()
|
||||
after := now.Add(time.Hour).UnixMilli()
|
||||
before := now.Add(-48 * time.Hour).UnixMilli()
|
||||
|
||||
list := []Outage{
|
||||
{Address: "A", StartDate: itoa(within), EndDate: itoa(after)}, // current
|
||||
{Address: "B", StartDate: itoa(before), EndDate: itoa(before)}, // past
|
||||
{Address: "C", StartDate: "", EndDate: ""}, // undated -> kept
|
||||
}
|
||||
got := Current(list, now, loc)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("expected 2 current outages, got %d: %+v", len(got), got)
|
||||
}
|
||||
}
|
||||
|
||||
func itoa(v int64) string {
|
||||
return strconv.FormatInt(v, 10)
|
||||
}
|
||||
Reference in New Issue
Block a user