addrkit
3be331bb65
- domain: address normalization/parsing, matching with explicit statuses - adapters: SQLite registry store, ArcGIS outages client, optional LLM rewriter, HTTP API - seed pipeline with duplicate handling and unparsed reporting - CLI (seed/verify/resolve), systemd unit, OpenAPI contract, tests
103 lines
2.6 KiB
Go
103 lines
2.6 KiB
Go
package address
|
||
|
||
import (
|
||
"strings"
|
||
"unicode"
|
||
|
||
"golang.org/x/text/cases"
|
||
"golang.org/x/text/unicode/norm"
|
||
)
|
||
|
||
// fold is a Unicode-aware case folder. It lower-cases Cyrillic and Latin
|
||
// alike, which is what we want for matching.
|
||
var fold = cases.Fold()
|
||
|
||
// NormalizeText folds a free-form string into a canonical form suitable for
|
||
// parsing and matching.
|
||
//
|
||
// The transformation is intentionally lossy:
|
||
// - Unicode NFKC (so lookalike characters collapse);
|
||
// - case folding (so "СТАНИЦИОННАЯ" == "станционная");
|
||
// - "ё" -> "е" (users and ASR disagree about the diaeresis);
|
||
// - dashes and various Unicode spaces are unified;
|
||
// - punctuation that only separates tokens becomes a space;
|
||
// - runs of whitespace collapse to a single space.
|
||
//
|
||
// NormalizeText is idempotent: NormalizeText(NormalizeText(s)) == NormalizeText(s).
|
||
func NormalizeText(s string) string {
|
||
s = norm.NFKC.String(s)
|
||
s = fold.String(s)
|
||
s = strings.ReplaceAll(s, "ё", "е")
|
||
s = mapSeparators(s)
|
||
s = strings.Join(strings.Fields(s), " ")
|
||
return strings.TrimSpace(s)
|
||
}
|
||
|
||
// mapSeparators unifies characters that carry no semantic meaning for address
|
||
// matching. It never removes digits, letters or the '/' house separator.
|
||
func mapSeparators(s string) string {
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
for _, r := range s {
|
||
switch {
|
||
case r == '\u00a0' || unicode.IsSpace(r):
|
||
b.WriteByte(' ')
|
||
case isDash(r):
|
||
b.WriteByte('-')
|
||
case isSeparatorPunct(r):
|
||
b.WriteByte(' ')
|
||
default:
|
||
b.WriteRune(r)
|
||
}
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// isDash reports whether r is one of the many dash-like code points that users
|
||
// and copy-paste bring in: "2–я" with an en dash is still an ordinal.
|
||
func isDash(r rune) bool {
|
||
switch r {
|
||
case '-', '\u2010', '\u2011', '\u2012', '\u2013', '\u2014', '\u2212':
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// isSeparatorPunct reports whether r separates tokens. Dots are included so
|
||
// that abbreviations like "ул." normalize to their stem ("ул").
|
||
func isSeparatorPunct(r rune) bool {
|
||
switch r {
|
||
case '.', ',', ';', '(', ')', '"', '\'', '«', '»', ':', '!', '?':
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// isDigits reports whether s is a non-empty run of ASCII digits.
|
||
func isDigits(s string) bool {
|
||
if s == "" {
|
||
return false
|
||
}
|
||
for _, r := range s {
|
||
if r < '0' || r > '9' {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// trimLeadingZeros removes redundant leading zeros from a numeric token:
|
||
// "007" -> "7". It leaves non-numeric strings untouched.
|
||
func trimLeadingZeros(s string) string {
|
||
if !isDigits(s) {
|
||
return s
|
||
}
|
||
t := strings.TrimLeft(s, "0")
|
||
if t == "" {
|
||
return "0"
|
||
}
|
||
return t
|
||
}
|