refactor!: make addrkit a multi-city address normalizer

- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
2026-10-27 14:35:51 +07:00
parent a0b4cb2b14
commit 17ff5c591f
38 changed files with 2978 additions and 1164 deletions
@@ -24,42 +24,121 @@ var (
 	// Trailing apartment part: " кв 5", " квартира 12", " кв. 7а". Removed
 	// before house parsing so it cannot leak into the street name.
 	reApartment = regexp.MustCompile(`(?:^|\s)(?:квартира|кв)\.?(?:\s+\d+[а-я]?)?\s*$`)
+
+	// Spoken fractions: "18 дробь 1" is the same house unit as "18/1".
+	reFractionWord = regexp.MustCompile(`\s+(?:дробь|дробью)\s+`)
 )

-// Parse decomposes a free-form address into its structured components. It is
-// total: unparseable input still yields a non-nil ParsedAddress whose street
-// part is best-effort and whose HasHouse is false.
+// Parser turns free-form address strings into structured addresses using a
+// deployment-specific Dictionary. It is immutable and safe for concurrent use.
+type Parser struct {
+	dict            *Dictionary
+	defaultCityName string
+	defaultCityKey  string
+}
+
+// NewParser builds a parser. defaultCity is used when the query does not name a
+// city and the dictionary has no match; pass an empty string for deployments
+// where the city must be present in every query.
+func NewParser(dict *Dictionary, defaultCity string) *Parser {
+	if dict == nil {
+		dict = DefaultDictionary()
+	}
+	p := &Parser{}
+	if name := strings.TrimSpace(defaultCity); name != "" {
+		dict = dict.withDefaultCity(name)
+		p.defaultCityName = name
+		p.defaultCityKey = NormalizeText(name)
+	}
+	p.dict = dict
+	return p
+}
+
+// defaultParser is used by the package-level Parse convenience function. It has
+// no cities and is therefore fully backward compatible: callers that need city
+// awareness construct their own Parser.
+var defaultParser = NewParser(DefaultDictionary(), "")
+
+// DefaultParser returns the shared city-less parser.
+func DefaultParser() *Parser {
+	return defaultParser
+}
+
+// Parse decomposes a free-form address with the default (city-less) parser.
 func Parse(raw string) ParsedAddress {
+	return defaultParser.Parse(raw)
+}
+
+// Parse decomposes a free-form address into its structured components. It is
+// total: unparseable input still yields a usable ParsedAddress whose street
+// part is best-effort and whose HasHouse is false.
+func (p *Parser) Parse(raw string) ParsedAddress {
+	return p.parse(raw, "")
+}
+
+// ParseInCity parses raw and forces cityName as the result's city. It is used
+// by the seed pipeline, where the city is an explicit field of the source row
+// and must win over whatever the address text contains.
+func (p *Parser) ParseInCity(raw, cityName string) ParsedAddress {
+	return p.parse(raw, cityName)
+}
+
+// MatchCity reports whether query is a city name/alias and returns its
+// canonical form. It does not guess a city from an arbitrary address.
+func (p *Parser) MatchCity(query string) (City, bool) {
+	return p.dict.MatchCity(query)
+}
+
+func (p *Parser) parse(raw, forcedCity string) ParsedAddress {
 	normalized := NormalizeText(raw)
 	// A trailing apartment ("кв 5") is not part of the address identity.
 	normalized = strings.TrimSpace(reApartment.ReplaceAllString(normalized, ""))
-	p := ParsedAddress{Raw: raw, Normalized: normalized}
+	// Collapse the spoken fraction marker to the canonical separator before
+	// the right-to-left house parser runs: "18 дробь 1" -> "18/1".
+	normalized = reFractionWord.ReplaceAllString(normalized, "/")
+	parsed := ParsedAddress{Raw: raw, Normalized: normalized}

 	rest := normalized
+	if cityName, cityKey, restTokens, ok := p.dict.matchCity(strings.Fields(rest)); ok {
+		parsed.City = cityName
+		parsed.CityKey = cityKey
+		rest = strings.Join(restTokens, " ")
+	}
+	if forcedCity = strings.TrimSpace(forcedCity); forcedCity != "" {
+		parsed.City = forcedCity
+		parsed.CityKey = NormalizeText(forcedCity)
+	} else if parsed.City == "" && p.defaultCityName != "" {
+		// A single-city deployment can omit the city in queries. In a
+		// multi-city deployment default_city should stay empty: guessing a
+		// city is exactly the kind of silent error this service avoids.
+		parsed.City = p.defaultCityName
+		parsed.CityKey = p.defaultCityKey
+	}
+
 	if g, start, ok := matchSuffix(reStructure, rest); ok {
-		p.Structure = trimLeadingZeros(g[1])
+		parsed.Structure = trimLeadingZeros(g[1])
 		rest = strings.TrimSpace(rest[:start])
 	}
 	if g, start, ok := matchSuffix(reBuilding, rest); ok {
 		// The regex also matches a bare "к"; only accept it when a digit or a
 		// letter actually follows, otherwise it is not a corpus marker.
 		if g[1] != "" || g[2] != "" {
-			p.Building = trimLeadingZeros(g[1])
-			p.BuildingLetter = firstNonEmpty(g[2], g[4])
-			p.BuildingFrac = trimLeadingZeros(g[3])
+			parsed.Building = trimLeadingZeros(g[1])
+			parsed.BuildingLetter = firstNonEmpty(g[2], g[4])
+			parsed.BuildingFrac = trimLeadingZeros(g[3])
 			rest = strings.TrimSpace(rest[:start])
 		}
 	}
 	if g, start, ok := matchSuffix(reHouse, rest); ok {
-		p.House = trimLeadingZeros(g[1])
-		p.HouseLetter = firstNonEmpty(g[2], g[4])
-		p.HouseFrac = trimLeadingZeros(g[3])
-		p.HasHouse = true
+		parsed.House = trimLeadingZeros(g[1])
+		parsed.HouseLetter = firstNonEmpty(g[2], g[4])
+		parsed.HouseFrac = trimLeadingZeros(g[3])
+		parsed.HasHouse = true
 		rest = strings.TrimSpace(rest[:start])
 	}

-	parseStreet(&p, rest)
-	return p
+	p.parseStreet(&parsed, rest)
+	return parsed
 }

 // firstNonEmpty returns the first non-empty argument. House/building letters
@@ -73,8 +152,8 @@ func firstNonEmpty(values ...string) string {
 	return ""
 }

-// parseStreet fills the street-related fields of p from the leftover string
-// (the address with its house block removed).
+// parseStreet fills the street-related fields of out from the leftover string
+// (the address with its city and house block removed).
 //
 // All recognised type tokens are removed from the name. This matters for
 // registry rows such as "Гусинобродский Тракт улица 21", where both "Тракт"
@@ -82,7 +161,7 @@ func firstNonEmpty(values ...string) string {
 // says "Гусинобродский 21". The first type token becomes the canonical type.
 // If removing them would leave the name empty (a street literally called
 // "Набережная"), the type token is kept as the name.
-func parseStreet(p *ParsedAddress, s string) {
+func (p *Parser) parseStreet(out *ParsedAddress, s string) {
 	tokens := strings.Fields(s)
 	name := make([]string, 0, len(tokens))
 	streetType := ""
@@ -90,10 +169,10 @@ func parseStreet(p *ParsedAddress, s string) {
 	for _, tok := range tokens {
 		// Filler tokens ("г", "город", "дом", "область", ...) never belong to
 		// the street name.
-		if fillerWords[tok] {
+		if p.dict.isFiller(tok) {
 			continue
 		}
-		if t, ok := streetTypes[tok]; ok {
+		if t, ok := p.dict.streetType(tok); ok {
 			if streetType == "" {
 				streetType = t
 			}
@@ -102,55 +181,55 @@ func parseStreet(p *ParsedAddress, s string) {
 		name = append(name, tok)
 	}

-	p.StreetType = streetType
+	out.StreetType = streetType
 	if len(name) == 0 && streetType != "" {
 		name = []string{streetType}
 	}
 	if len(name) == 0 {
-		p.StreetKey = streetKey("", "")
+		out.StreetKey = streetKey(out.CityKey, "", "")
 		return
 	}

-	name = extractOrdinal(p, name)
-	p.NameDisplay = strings.Join(name, " ")
+	name = p.extractOrdinal(out, name)
+	out.NameDisplay = strings.Join(name, " ")

 	sorted := append([]string(nil), name...)
 	sort.Strings(sorted)
-	p.NameKey = strings.Join(sorted, " ")
-	p.StreetKey = streetKey(p.Ordinal, p.NameKey)
+	out.NameKey = strings.Join(sorted, " ")
+	out.StreetKey = streetKey(out.CityKey, out.Ordinal, out.NameKey)
 }

 // extractOrdinal recognises an ordinal and removes it from the name tokens.
 // It handles both the usual leading form ("2-я Сухарная") and an ordinal that
 // appears after the name ("Сухарная 2-я улица"), which both people and models
 // produce.
-func extractOrdinal(p *ParsedAddress, tokens []string) []string {
+func (p *Parser) extractOrdinal(out *ParsedAddress, tokens []string) []string {
 	if len(tokens) == 0 {
 		return tokens
 	}

 	// Leading forms first: they are by far the most common.
 	if m := reOrdinalToken.FindStringSubmatch(tokens[0]); m != nil {
-		p.Ordinal = normalizeOrdinal(m[1])
+		out.Ordinal = normalizeOrdinal(m[1])
 		return tokens[1:]
 	}
-	if isDigits(tokens[0]) && len(tokens) >= 2 && ordinalSuffixes[tokens[1]] {
-		p.Ordinal = normalizeOrdinal(tokens[0])
+	if isDigits(tokens[0]) && len(tokens) >= 2 && p.dict.isOrdinalSuffix(tokens[1]) {
+		out.Ordinal = normalizeOrdinal(tokens[0])
 		return tokens[2:]
 	}
-	if d, ok := ordinalWords[tokens[0]]; ok {
-		p.Ordinal = normalizeOrdinal(d)
+	if d, ok := p.dict.ordinalWord(tokens[0]); ok {
+		out.Ordinal = normalizeOrdinal(d)
 		return tokens[1:]
 	}

 	// Otherwise look for an ordinal anywhere among the name tokens.
 	for i, tok := range tokens {
 		if m := reOrdinalToken.FindStringSubmatch(tok); m != nil {
-			p.Ordinal = normalizeOrdinal(m[1])
+			out.Ordinal = normalizeOrdinal(m[1])
 			return removeRange(tokens, i, i+1)
 		}
-		if isDigits(tok) && i+1 < len(tokens) && ordinalSuffixes[tokens[i+1]] {
-			p.Ordinal = normalizeOrdinal(tok)
+		if isDigits(tok) && i+1 < len(tokens) && p.dict.isOrdinalSuffix(tokens[i+1]) {
+			out.Ordinal = normalizeOrdinal(tok)
 			return removeRange(tokens, i, i+2)
 		}
 	}