package address import ( "regexp" "strings" "unicode" "golang.org/x/text/cases" "golang.org/x/text/unicode/norm" ) // fold is a Unicode-aware case folder. It lower-cases Cyrillic and Latin // alike, which is what we want for matching. var fold = cases.Fold() // reTrailingParenthetical matches a trailing "(...)" group, e.g. the district // suffix that source systems append: "Медкадры улица, 1 (Заельцовский // район)". Such administrative noise must not be treated as part of the house. var reTrailingParenthetical = regexp.MustCompile(`\s*\([^)]*\)\s*$`) // NormalizeText folds a free-form string into a canonical form suitable for // parsing and matching. // // The transformation is intentionally lossy: // - Unicode NFKC (so lookalike characters collapse); // - case folding (so "СТАНИЦИОННАЯ" == "станционная"); // - "ё" -> "е" (users and ASR disagree about the diaeresis); // - dashes and various Unicode spaces are unified; // - punctuation that only separates tokens becomes a space; // - runs of whitespace collapse to a single space. // // NormalizeText is idempotent: NormalizeText(NormalizeText(s)) == NormalizeText(s). func NormalizeText(s string) string { s = norm.NFKC.String(s) s = fold.String(s) s = strings.ReplaceAll(s, "ё", "е") s = stripTrailingParenthetical(s) s = mapSeparators(s) s = strings.Join(strings.Fields(s), " ") return strings.TrimSpace(s) } // stripTrailingParenthetical removes a single trailing "(...)" group. It runs // before separator mapping because that step turns parentheses into spaces and // the group would no longer be recognisable. func stripTrailingParenthetical(s string) string { return strings.TrimSpace(reTrailingParenthetical.ReplaceAllString(s, "")) } // mapSeparators unifies characters that carry no semantic meaning for address // matching. It never removes digits, letters or the '/' house separator. func mapSeparators(s string) string { var b strings.Builder b.Grow(len(s)) for _, r := range s { switch { case r == '\u00a0' || unicode.IsSpace(r): b.WriteByte(' ') case isDash(r): b.WriteByte('-') case isSeparatorPunct(r): b.WriteByte(' ') default: b.WriteRune(r) } } return b.String() } // isDash reports whether r is one of the many dash-like code points that users // and copy-paste bring in: "2–я" with an en dash is still an ordinal. func isDash(r rune) bool { switch r { case '-', '\u2010', '\u2011', '\u2012', '\u2013', '\u2014', '\u2212': return true default: return false } } // isSeparatorPunct reports whether r separates tokens. Dots are included so // that abbreviations like "ул." normalize to their stem ("ул"). func isSeparatorPunct(r rune) bool { switch r { case '.', ',', ';', '(', ')', '"', '\'', '«', '»', ':', '!', '?': return true default: return false } } // isDigits reports whether s is a non-empty run of ASCII digits. func isDigits(s string) bool { if s == "" { return false } for _, r := range s { if r < '0' || r > '9' { return false } } return true } // trimLeadingZeros removes redundant leading zeros from a numeric token: // "007" -> "7". It leaves non-numeric strings untouched. func trimLeadingZeros(s string) string { if !isDigits(s) { return s } t := strings.TrimLeft(s, "0") if t == "" { return "0" } return t }