refactor!: make addrkit a multi-city address normalizer
- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
Maksim Totmin
+87
-20
@@ -4,11 +4,15 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
|
||||
)
|
||||
|
||||
// Config is the whole service configuration.
|
||||
@@ -21,25 +25,24 @@ type Config struct {
|
||||
LogLevel string `json:"log_level"`
|
||||
|
||||
Resolve ResolveConfig `json:"resolve"`
|
||||
Portal PortalConfig `json:"portal"`
|
||||
LLM LLMConfig `json:"llm"`
|
||||
|
||||
// Cities is the dictionary of cities this deployment understands. A
|
||||
// single-city deployment lists one city and may set DefaultCity so that
|
||||
// queries like "ленина 5" work without naming it.
|
||||
Cities []address.City `json:"cities"`
|
||||
DefaultCity string `json:"default_city"`
|
||||
|
||||
LLM LLMConfig `json:"llm"`
|
||||
}
|
||||
|
||||
// ResolveConfig tunes the matching engine.
|
||||
// ResolveConfig tunes the matching engine. Defaults come from
|
||||
// match.DefaultOptions so there is exactly one source of truth.
|
||||
type ResolveConfig struct {
|
||||
FuzzyThreshold float64 `json:"fuzzy_threshold"`
|
||||
AmbiguityMargin float64 `json:"ambiguity_margin"`
|
||||
MaxCandidates int `json:"max_candidates"`
|
||||
}
|
||||
|
||||
// PortalConfig configures the external outages portal (ArcGIS). When URL is
|
||||
// empty the "events by house" module is disabled and only /v1/resolve works.
|
||||
type PortalConfig struct {
|
||||
URL string `json:"url"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
Retries int `json:"retries"`
|
||||
}
|
||||
|
||||
// LLMConfig configures the optional, disabled-by-default query rewriter.
|
||||
type LLMConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
@@ -63,17 +66,14 @@ type LLMConfig struct {
|
||||
|
||||
// Default returns a config with sane defaults for the non-required fields.
|
||||
func Default() Config {
|
||||
resolve := match.DefaultOptions()
|
||||
return Config{
|
||||
Addr: ":6171",
|
||||
LogLevel: "info",
|
||||
Resolve: ResolveConfig{
|
||||
FuzzyThreshold: 0.82,
|
||||
AmbiguityMargin: 0.03,
|
||||
MaxCandidates: 5,
|
||||
},
|
||||
Portal: PortalConfig{
|
||||
TimeoutSeconds: 2,
|
||||
Retries: 1,
|
||||
FuzzyThreshold: resolve.FuzzyThreshold,
|
||||
AmbiguityMargin: resolve.AmbiguityMargin,
|
||||
MaxCandidates: resolve.MaxCandidates,
|
||||
},
|
||||
LLM: LLMConfig{
|
||||
TimeoutSeconds: 3,
|
||||
@@ -91,6 +91,9 @@ func Load(path string) (Config, error) {
|
||||
}
|
||||
cfg := Default()
|
||||
dec := json.NewDecoder(strings.NewReader(expandEnv(string(raw))))
|
||||
// A typo in a key (fuzzy_treshold, data_pth, portal, ...) must not silently
|
||||
// fall back to a default. Fail closed and let the operator fix the config.
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&cfg); err != nil {
|
||||
return Config{}, fmt.Errorf("parse config: %w", err)
|
||||
}
|
||||
@@ -107,6 +110,25 @@ func Load(path string) (Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// LoadCitiesFile reads a standalone JSON array of cities. It is used by the
|
||||
// seed CLI so that seeding and serving share the same city vocabulary.
|
||||
func LoadCitiesFile(path string) ([]address.City, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read cities: %w", err)
|
||||
}
|
||||
var cities []address.City
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&cities); err != nil {
|
||||
return nil, fmt.Errorf("parse cities: %w", err)
|
||||
}
|
||||
if err := validateCities(cities, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cities, nil
|
||||
}
|
||||
|
||||
func (c Config) validate() error {
|
||||
if c.Addr == "" {
|
||||
return fmt.Errorf("config: addr is required")
|
||||
@@ -114,11 +136,56 @@ func (c Config) validate() error {
|
||||
if c.DataPath == "" {
|
||||
return fmt.Errorf("config: data_path is required")
|
||||
}
|
||||
if c.Resolve.FuzzyThreshold <= 0 || c.Resolve.FuzzyThreshold > 1 {
|
||||
return fmt.Errorf("config: resolve.fuzzy_threshold must be in (0, 1]")
|
||||
}
|
||||
if c.Resolve.AmbiguityMargin < 0 || c.Resolve.AmbiguityMargin > 1 {
|
||||
return fmt.Errorf("config: resolve.ambiguity_margin must be in [0, 1]")
|
||||
}
|
||||
if c.Resolve.MaxCandidates < 1 {
|
||||
return fmt.Errorf("config: resolve.max_candidates must be at least 1")
|
||||
}
|
||||
if c.LLM.Enabled && c.LLM.URL == "" {
|
||||
return fmt.Errorf("config: llm.url is required when llm.enabled is true")
|
||||
}
|
||||
// portal.url is optional: without it the "events by house" module is
|
||||
// disabled and only /v1/resolve is served.
|
||||
return validateCities(c.Cities, c.DefaultCity)
|
||||
}
|
||||
|
||||
// validateCities checks the city dictionary for the mistakes that would make
|
||||
// matching silently ambiguous: empty names, duplicate cities and an alias
|
||||
// claimed by two different cities.
|
||||
func validateCities(cities []address.City, defaultCity string) error {
|
||||
seenNames := make(map[string]string)
|
||||
seenAliases := make(map[string]string)
|
||||
|
||||
for _, city := range cities {
|
||||
name := strings.TrimSpace(city.Name)
|
||||
key := address.NormalizeText(name)
|
||||
if key == "" {
|
||||
return fmt.Errorf("config: cities[].name must not be empty")
|
||||
}
|
||||
if prev, ok := seenNames[key]; ok {
|
||||
return fmt.Errorf("config: duplicate city %q and %q", prev, name)
|
||||
}
|
||||
seenNames[key] = name
|
||||
|
||||
aliases := append([]string{name}, city.Aliases...)
|
||||
for _, alias := range aliases {
|
||||
norm := address.NormalizeText(alias)
|
||||
if norm == "" {
|
||||
continue
|
||||
}
|
||||
if owner, ok := seenAliases[norm]; ok && owner != key {
|
||||
return fmt.Errorf("config: alias %q is claimed by both %q and %q", alias, owner, key)
|
||||
}
|
||||
seenAliases[norm] = key
|
||||
}
|
||||
}
|
||||
if defaultCity = strings.TrimSpace(defaultCity); defaultCity != "" && len(cities) > 0 {
|
||||
if _, ok := seenNames[address.NormalizeText(defaultCity)]; !ok {
|
||||
return fmt.Errorf("config: default_city %q is not listed in cities", defaultCity)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
|
||||
)
|
||||
|
||||
func writeFile(t *testing.T, dir, name, content string) string {
|
||||
@@ -19,7 +21,6 @@ func baseConfig(promptFields string) string {
|
||||
return `{
|
||||
"addr": ":6171",
|
||||
"data_path": "registry.db",
|
||||
"portal": { "url": "http://portal" },
|
||||
"llm": { "enabled": true, "url": "http://llm"` + promptFields + ` }
|
||||
}`
|
||||
}
|
||||
@@ -60,11 +61,90 @@ func TestLoadMissingSystemPromptFileFailsClosed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPortalURLOptional(t *testing.T) {
|
||||
func TestDefaultsMatchMatcher(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// No portal.url: the events module is disabled, the core still loads.
|
||||
path := writeFile(t, dir, "config.json", `{"addr": ":6171", "data_path": "x.db"}`)
|
||||
if _, err := Load(path); err != nil {
|
||||
t.Fatalf("portal.url must be optional: %v", err)
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
want := match.DefaultOptions()
|
||||
if cfg.Resolve.FuzzyThreshold != want.FuzzyThreshold ||
|
||||
cfg.Resolve.AmbiguityMargin != want.AmbiguityMargin ||
|
||||
cfg.Resolve.MaxCandidates != want.MaxCandidates {
|
||||
t.Fatalf("config defaults %+v do not match matcher defaults %+v", cfg.Resolve, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectsBadResolveRanges(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cases := map[string]string{
|
||||
"threshold": `{"addr":":6171","data_path":"x.db","resolve":{"fuzzy_threshold":0}}`,
|
||||
"margin": `{"addr":":6171","data_path":"x.db","resolve":{"ambiguity_margin":2}}`,
|
||||
"candidates": `{"addr":":6171","data_path":"x.db","resolve":{"max_candidates":0}}`,
|
||||
}
|
||||
for name, body := range cases {
|
||||
path := writeFile(t, dir, name+".json", body)
|
||||
if _, err := Load(path); err == nil {
|
||||
t.Errorf("%s: expected validation error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCitiesAndDefaultCity(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := writeFile(t, dir, "config.json", `{
|
||||
"addr": ":6171", "data_path": "x.db",
|
||||
"cities": [
|
||||
{"name": "Новосибирск", "aliases": ["нск", "новосибирская область"]},
|
||||
{"name": "Бердск"}
|
||||
],
|
||||
"default_city": "Новосибирск"
|
||||
}`)
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if len(cfg.Cities) != 2 || cfg.DefaultCity != "Новосибирск" {
|
||||
t.Fatalf("cities = %+v, default = %q", cfg.Cities, cfg.DefaultCity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectsUnknownDefaultCity(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := writeFile(t, dir, "config.json", `{
|
||||
"addr": ":6171", "data_path": "x.db",
|
||||
"cities": [{"name": "Бердск"}],
|
||||
"default_city": "Новосибирск"
|
||||
}`)
|
||||
if _, err := Load(path); err == nil {
|
||||
t.Fatal("expected error for default_city absent from cities")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCitiesFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := writeFile(t, dir, "cities.json", `[
|
||||
{"name": "Новосибирск", "aliases": ["нск"]},
|
||||
{"name": "Бердск"}
|
||||
]`)
|
||||
cities, err := LoadCitiesFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadCitiesFile: %v", err)
|
||||
}
|
||||
if len(cities) != 2 || cities[0].Name != "Новосибирск" {
|
||||
t.Fatalf("cities = %+v", cities)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectsUnknownConfigFields(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// "portal" was removed with the events module; silently ignoring it would
|
||||
// hide a stale deployment config.
|
||||
path := writeFile(t, dir, "config.json",
|
||||
`{"addr":":6171","data_path":"x.db","portal":{"url":"http://example"}}`)
|
||||
if _, err := Load(path); err == nil {
|
||||
t.Fatal("expected error for unknown config field")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user