Maksim Totmin
17ff5c591f
- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
117 lines
3.3 KiB
Go
117 lines
3.3 KiB
Go
package address
|
||
|
||
import (
|
||
"regexp"
|
||
"strings"
|
||
"unicode"
|
||
|
||
"golang.org/x/text/cases"
|
||
"golang.org/x/text/unicode/norm"
|
||
)
|
||
|
||
// fold is a Unicode-aware case folder. It lower-cases Cyrillic and Latin
|
||
// alike, which is what we want for matching.
|
||
var fold = cases.Fold()
|
||
|
||
// reTrailingParenthetical matches a trailing "(...)" group, e.g. the district
|
||
// suffix that source systems append: "Медкадры улица, 1 (Заельцовский
|
||
// район)". Such administrative noise must not be treated as part of the house.
|
||
var reTrailingParenthetical = regexp.MustCompile(`\s*\([^)]*\)\s*$`)
|
||
|
||
// NormalizeText folds a free-form string into a canonical form suitable for
|
||
// parsing and matching.
|
||
//
|
||
// The transformation is intentionally lossy:
|
||
// - Unicode NFKC (so lookalike characters collapse);
|
||
// - case folding (so "СТАНИЦИОННАЯ" == "станционная");
|
||
// - "ё" -> "е" (users and ASR disagree about the diaeresis);
|
||
// - dashes and various Unicode spaces are unified;
|
||
// - punctuation that only separates tokens becomes a space;
|
||
// - runs of whitespace collapse to a single space.
|
||
//
|
||
// NormalizeText is idempotent: NormalizeText(NormalizeText(s)) == NormalizeText(s).
|
||
func NormalizeText(s string) string {
|
||
s = norm.NFKC.String(s)
|
||
s = fold.String(s)
|
||
s = strings.ReplaceAll(s, "ё", "е")
|
||
s = stripTrailingParenthetical(s)
|
||
s = mapSeparators(s)
|
||
s = strings.Join(strings.Fields(s), " ")
|
||
return strings.TrimSpace(s)
|
||
}
|
||
|
||
// stripTrailingParenthetical removes a single trailing "(...)" group. It runs
|
||
// before separator mapping because that step turns parentheses into spaces and
|
||
// the group would no longer be recognisable.
|
||
func stripTrailingParenthetical(s string) string {
|
||
return strings.TrimSpace(reTrailingParenthetical.ReplaceAllString(s, ""))
|
||
}
|
||
|
||
// mapSeparators unifies characters that carry no semantic meaning for address
|
||
// matching. It never removes digits, letters or the '/' house separator.
|
||
func mapSeparators(s string) string {
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
for _, r := range s {
|
||
switch {
|
||
case r == '\u00a0' || unicode.IsSpace(r):
|
||
b.WriteByte(' ')
|
||
case isDash(r):
|
||
b.WriteByte('-')
|
||
case isSeparatorPunct(r):
|
||
b.WriteByte(' ')
|
||
default:
|
||
b.WriteRune(r)
|
||
}
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// isDash reports whether r is one of the many dash-like code points that users
|
||
// and copy-paste bring in: "2–я" with an en dash is still an ordinal.
|
||
func isDash(r rune) bool {
|
||
switch r {
|
||
case '-', '\u2010', '\u2011', '\u2012', '\u2013', '\u2014', '\u2212':
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// isSeparatorPunct reports whether r separates tokens. Dots are included so
|
||
// that abbreviations like "ул." normalize to their stem ("ул").
|
||
func isSeparatorPunct(r rune) bool {
|
||
switch r {
|
||
case '.', ',', ';', '(', ')', '"', '\'', '«', '»', ':', '!', '?':
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// isDigits reports whether s is a non-empty run of ASCII digits.
|
||
func isDigits(s string) bool {
|
||
if s == "" {
|
||
return false
|
||
}
|
||
for _, r := range s {
|
||
if r < '0' || r > '9' {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// trimLeadingZeros removes redundant leading zeros from a numeric token:
|
||
// "007" -> "7". It leaves non-numeric strings untouched.
|
||
func trimLeadingZeros(s string) string {
|
||
if !isDigits(s) {
|
||
return s
|
||
}
|
||
t := strings.TrimLeft(s, "0")
|
||
if t == "" {
|
||
return "0"
|
||
}
|
||
return t
|
||
}
|