en2zmax/addrkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor!: make addrkit a multi-city address normalizer

Browse Source 
- remove the events/outages module, ArcGIS adapter, portal config and event statuses
- add a configurable city dictionary/parser and make City part of StreetKey
- seed: read city/city_name, namespace IDs by city, keep structures distinct
- fix reliability: schema v2 with fail-closed load, deterministic ordering,
  input limits, config validation, atomic DB publish, LLM response caps
- add read-only shadow mode comparing the old ES path with addrkit
- breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
Maksim Totmin
2026-10-27 14:35:51 +07:00
parent a0b4cb2b14
commit 17ff5c591f
38 changed files with 2978 additions and 1164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/adapters/arcgis/client.go
-256
View File
@@ -1,256 +0,0 @@
// Package arcgis is the adapter for the external outages portal exposed as an
// ArcGIS MapServer /query endpoint.
//
// The portal only supports substring (LIKE) filtering, so precision is
// guaranteed locally: whatever the portal returns is re-filtered by the
// outages domain using the parsed house unit. The client adds the reliability
// the portal lacks: a hard timeout, bounded retries with backoff and context
// cancellation.
package arcgis
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
"git.totmin.ru/en2zmax/addrkit/internal/domain/outages"
)
// reDistrictSuffix matches the administrative suffix the portal appends to its
// display address, e.g. "Медкадры улица, 1 (Заельцовский район)". It must be
// removed before the address is parsed, otherwise the trailing district text
// hides the house number. The district is available separately as a field.
var reDistrictSuffix = regexp.MustCompile(`\s*\([^)]*\)\s*$`)
// cleanAddress strips the portal's district suffix and surrounding spaces.
func cleanAddress(s string) string {
return strings.TrimSpace(reDistrictSuffix.ReplaceAllString(s, ""))
}
// Client is a bounded HTTP client for the portal.
type Client struct {
baseURL string
http *http.Client
retries int
}
// New builds a client. timeout applies to a single attempt; retries is the
// number of *additional* attempts after the first.
func New(baseURL string, timeout time.Duration, retries int) *Client {
return &Client{
baseURL: baseURL,
http: &http.Client{Timeout: timeout},
retries: retries,
}
}
// Query asks the portal for outages matching the canonical address. The
// returned records are raw portal rows; callers must still filter by house.
func (c *Client) Query(ctx context.Context, canonical string) ([]outages.Outage, error) {
params := url.Values{}
params.Set("where", "address like '"+likePattern(canonical)+"'")
params.Set("outFields", "")
params.Set("returnGeometry", "false")
params.Set("f", "json")
var lastErr error
for attempt := 0; attempt <= c.retries; attempt++ {
if attempt > 0 {
backoff := time.Duration(attempt) * 150 * time.Millisecond
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(backoff):
}
}
rows, err := c.do(ctx, params)
if err == nil {
return rows, nil
}
lastErr = err
}
return nil, fmt.Errorf("portal query failed after %d attempts: %w", c.retries+1, lastErr)
}
func (c *Client) do(ctx context.Context, params url.Values) ([]outages.Outage, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"?"+params.Encode(), nil)
if err != nil {
return nil, fmt.Errorf("build request: %w", err)
}
resp, err := c.http.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("portal status %d", resp.StatusCode)
}
// Bound the response size to protect the service from a misbehaving portal.
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if err != nil {
return nil, err
}
return parseResponse(body)
}
// portalResponse mirrors the subset of the ArcGIS /query envelope we use.
type portalResponse struct {
Features []struct {
Value string `json:"value"`
Attributes map[string]any `json:"attributes"`
} `json:"features"`
Fields []struct {
Name string `json:"name"`
Domain struct {
CodedValues []struct {
Code any `json:"code"`
Name string `json:"name"`
} `json:"codedValues"`
} `json:"domain"`
} `json:"fields"`
}
func parseResponse(body []byte) ([]outages.Outage, error) {
var pr portalResponse
if err := json.Unmarshal(body, &pr); err != nil {
return nil, fmt.Errorf("decode portal response: %w", err)
}
// Build code -> name maps from the coded-value domains.
maps := map[string]map[string]string{}
for _, f := range pr.Fields {
if len(f.Domain.CodedValues) == 0 {
continue
}
m := make(map[string]string, len(f.Domain.CodedValues))
for _, cv := range f.Domain.CodedValues {
m[codeKey(cv.Code)] = cv.Name
}
maps[f.Name] = m
}
out := make([]outages.Outage, 0, len(pr.Features))
for _, feat := range pr.Features {
addr := cleanAddress(addressOf(feat.Attributes, feat.Value))
o := outages.Outage{
Address: addr,
Type: lookup(maps, "type_id", feat.Attributes),
System: lookup(maps, "system_id", feat.Attributes),
Time: lookup(maps, "time_category", feat.Attributes),
StartDate: stringOf(feat.Attributes["start_date"]),
EndDate: stringOf(feat.Attributes["end_date"]),
}
o.Action = actionFor(o.System)
// Parse the address at the boundary so the domain matches on structure
// and never depends on this portal's text format.
o.Unit = outages.HouseUnitOf(address.Parse(addr))
out = append(out, o)
}
return out, nil
}
func addressOf(attrs map[string]any, fallback string) string {
if v, ok := attrs["address"]; ok {
if s := stringOf(v); s != "" {
return s
}
}
return fallback
}
func lookup(maps map[string]map[string]string, field string, attrs map[string]any) string {
m, ok := maps[field]
if !ok {
return stringOf(attrs[field])
}
return m[codeKey(attrs[field])]
}
// codeKey normalises a coded value coming from JSON (float64) or a string.
func codeKey(v any) string {
switch t := v.(type) {
case nil:
return ""
case string:
return t
case float64:
return strconv.FormatInt(int64(t), 10)
case json.Number:
return t.String()
default:
return fmt.Sprintf("%v", v)
}
}
// stringOf renders an attribute value as a string. The important case is
// numeric dates: fmt's %v turns a float64 into scientific notation ("1.79e+12")
// which the date parser cannot read, so integers are formatted explicitly.
func stringOf(v any) string {
switch t := v.(type) {
case nil:
return ""
case string:
return t
case float64:
return strconv.FormatInt(int64(t), 10)
case json.Number:
return t.String()
default:
return fmt.Sprintf("%v", v)
}
}
// actionFor maps a portal system name to the canonical action code used by the
// scenario (kept for parity with the existing integration).
func actionFor(system string) string {
switch system {
case "Горячее водоснабжение":
return "waterWarm"
case "Холодное водоснабжение и канализация", "Холодное водоснабжение":
return "waterCold"
case "Электроснабжение":
return "electricitySupply"
case "Газоснабжение":
return "gasSupply"
case "Теплоснабжение":
return "heating"
default:
return ""
}
}
// likePattern builds the substring pattern the portal supports. Only pure
// numeric tokens (house numbers) are padded with spaces, so "1" does not
// incidentally match "10". Ordinals ("3-я") and houses with a letter ("24а")
// are left as-is: padding an ordinal breaks the match when it starts the
// string, and precision is guaranteed by the local house filter anyway.
func likePattern(canonical string) string {
parts := strings.Fields(canonical)
for i, part := range parts {
if isPureDigits(part) {
parts[i] = " " + part + " "
}
}
return "%" + strings.Join(parts, "%") + "%"
}
func isPureDigits(s string) bool {
if s == "" {
return false
}
for _, r := range s {
if r < '0' || r > '9' {
return false
}
}
return true
}
internal/adapters/arcgis/client_test.go
-128
View File
@@ -1,128 +0,0 @@
package arcgis
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
const envelope = `{
"features": [
{"value": "Станционная улица 1/1", "attributes": {
"address": "Станционная улица 1/1",
"type_id": 1, "system_id": 5, "time_category": 2,
"start_date": "2026-10-01T00:00:00", "end_date": "2026-10-31T00:00:00"
}}
],
"fields": [
{"name": "type_id", "domain": {"codedValues": [{"code": 1, "name": "Плановое"}]}},
{"name": "system_id", "domain": {"codedValues": [{"code": 5, "name": "Теплоснабжение"}]}},
{"name": "time_category", "domain": {"codedValues": [{"code": 2, "name": "Текущие"}]}}
]
}`
func TestQueryParsesEnvelope(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.URL.Query().Get("f"); got != "json" {
t.Errorf("missing f=json, got %q", got)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(envelope))
}))
defer ts.Close()
c := New(ts.URL, time.Second, 0)
rows, err := c.Query(context.Background(), "Станционная улица 1/1")
if err != nil {
t.Fatalf("Query: %v", err)
}
if len(rows) != 1 {
t.Fatalf("rows = %d, want 1", len(rows))
}
got := rows[0]
if got.Type != "Плановое" || got.System != "Теплоснабжение" || got.Time != "Текущие" {
t.Errorf("coded values not resolved: %+v", got)
}
if got.Action != "heating" {
t.Errorf("Action = %q, want heating", got.Action)
}
}
func TestQueryRetriesThenFails(t *testing.T) {
var calls int
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
http.Error(w, "boom", http.StatusInternalServerError)
}))
defer ts.Close()
c := New(ts.URL, time.Second, 2)
if _, err := c.Query(context.Background(), "Станционная улица 1/1"); err == nil {
t.Fatal("expected error")
}
if calls != 3 {
t.Errorf("calls = %d, want 3 (1 + 2 retries)", calls)
}
}
// TestQueryStripsDistrictSuffix is a regression test: the portal appends the
// district in parentheses, which used to hide the house number from the parser.
func TestQueryStripsDistrictSuffix(t *testing.T) {
const env = `{"features":[{"value":"Медкадры улица, 1 (Заельцовский район)","attributes":{"address":"Медкадры улица, 1 (Заельцовский район)"}}]}`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(env))
}))
defer ts.Close()
c := New(ts.URL, time.Second, 0)
rows, err := c.Query(context.Background(), "Медкадры улица 1")
if err != nil {
t.Fatalf("Query: %v", err)
}
if len(rows) != 1 {
t.Fatalf("rows = %d, want 1", len(rows))
}
if rows[0].Address != "Медкадры улица, 1" {
t.Errorf("district suffix not stripped: %q", rows[0].Address)
}
if rows[0].Unit.House != "1" {
t.Errorf("house not parsed at the boundary: %+v", rows[0].Unit)
}
}
// TestQueryFormatsNumericDates guards against scientific notation in dates,
// which broke the "current outage" filter.
func TestQueryFormatsNumericDates(t *testing.T) {
const env = `{"features":[{"value":"Ленина улица, 1","attributes":{"address":"Ленина улица, 1","start_date":1790820000000,"end_date":1791799200000}}]}`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(env))
}))
defer ts.Close()
rows, err := New(ts.URL, time.Second, 0).Query(context.Background(), "Ленина улица 1")
if err != nil {
t.Fatalf("Query: %v", err)
}
if rows[0].StartDate != "1790820000000" || rows[0].EndDate != "1791799200000" {
t.Errorf("numeric dates must be integer-formatted: %q..%q", rows[0].StartDate, rows[0].EndDate)
}
}
// TestLikePattern guards the ordinal/house padding rule: padding an ordinal
// that starts the string breaks the portal match.
func TestLikePattern(t *testing.T) {
got := likePattern("3-я Механическая улица 24а")
if strings.Contains(got, " 3-я ") {
t.Errorf("ordinal must not be space-padded: %q", got)
}
if strings.Contains(got, " 24а ") {
t.Errorf("house with a letter must not be space-padded: %q", got)
}
if !strings.Contains(likePattern("Ленина улица 1"), "% 1 %") {
t.Errorf("pure numeric house must be space-padded: %q", likePattern("Ленина улица 1"))
}
}
internal/adapters/httpapi/middleware.go
+25 -3
View File
@@ -14,10 +14,13 @@ type ctxKey int
const requestIDKey ctxKey = 0
// requestIDMW assigns a request id (for correlation in logs) and echoes it.
// A client-supplied id is accepted only when it is short and well-formed; the
// header ends up in logs, so accepting arbitrary bytes would be an injection
// and log-flooding vector.
func requestIDMW(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
id := r.Header.Get("X-Request-Id")
if id == "" {
if !validRequestID(id) {
id = newRequestID()
}
w.Header().Set("X-Request-Id", id)
@@ -26,6 +29,23 @@ func requestIDMW(next http.Handler) http.Handler {
})
}
const maxRequestIDLen = 128
func validRequestID(id string) bool {
if id == "" || len(id) > maxRequestIDLen {
return false
}
for _, r := range id {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
case r == '-', r == '_', r == '.':
default:
return false
}
}
return true
}
// loggingMW logs one line per request with method, path, status and duration.
// It records no addresses (PII) — only routing metadata.
func loggingMW(log *slog.Logger) func(http.Handler) http.Handler {
@@ -45,8 +65,10 @@ func loggingMW(log *slog.Logger) func(http.Handler) http.Handler {
}
}
// timeoutMW bounds the whole request, so a slow portal cannot tie up a
// goroutine indefinitely.
// timeoutMW bounds the whole request by setting a context deadline. Handlers
// and adapters that talk to the network honour it; purely CPU-bound handlers
// are additionally bounded by input limits and the HTTP server's read/write
// timeouts in cmd/addrkitd.
func timeoutMW(d time.Duration) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
internal/adapters/httpapi/server.go
+66 -66
View File
@@ -1,52 +1,54 @@
// Package httpapi exposes the addrkit domain over a small, versioned JSON API.
//
// It contains no business logic: handlers decode requests, call the resolver
// and the outages source, and encode responses. Cross-cutting concerns
// (timeouts, recovery, request ids, logging) live in middleware.
// and encode responses. Cross-cutting concerns (timeouts, recovery, request
// ids, logging) live in middleware.
package httpapi
import (
"context"
"encoding/json"
"errors"
"log/slog"
"net/http"
"time"
"unicode/utf8"
"log/slog"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
"git.totmin.ru/en2zmax/addrkit/internal/domain/outages"
)
// Resolver is the read side of the domain (implemented by *match.Resolver).
type Resolver interface {
Resolve(ctx context.Context, raw string) match.Result
ResolveInCity(ctx context.Context, raw, city string) match.Result
MatchCity(query string) (string, bool)
Version() string
Ready() bool
Cities() []string
}
// maxAddressRunes bounds user input. A real address is far shorter; the limit
// keeps a pathological string from turning into a CPU-bound fuzzy scan.
const maxAddressRunes = 512
// maxCityQueryRunes bounds the separate city-only endpoint.
const maxCityQueryRunes = 128
// Server wires the domain to HTTP.
type Server struct {
resolver Resolver
outages outages.Source
log *slog.Logger
now func() time.Time
loc *time.Location
timeout time.Duration
}
// New builds a server. loc controls how outage dates are interpreted; it
// should be the operational timezone (Asia/Novosibirsk).
func New(resolver Resolver, src outages.Source, log *slog.Logger, loc *time.Location) *Server {
if loc == nil {
loc = time.UTC
// New builds a server.
func New(resolver Resolver, log *slog.Logger) *Server {
if log == nil {
log = slog.Default()
}
return &Server{
resolver: resolver,
outages: src,
log: log,
now: time.Now,
loc: loc,
timeout: 10 * time.Second,
}
}
@@ -55,16 +57,21 @@ func New(resolver Resolver, src outages.Source, log *slog.Logger, loc *time.Loca
func (s *Server) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("POST /v1/resolve", s.handleResolve)
mux.HandleFunc("POST /v1/outages", s.handleOutages)
mux.HandleFunc("POST /v1/city", s.handleCity)
mux.HandleFunc("GET /v1/cities", s.handleCities)
mux.HandleFunc("GET /v1/healthz", s.handleHealthz)
mux.HandleFunc("GET /v1/readyz", s.handleReadyz)
mux.HandleFunc("GET /v1/version", s.handleVersion)
return recoverMW(s.log)(requestIDMW(loggingMW(s.log)(timeoutMW(s.timeout)(mux))))
// Logging sits outside recovery so a panic is still recorded as a 500.
return requestIDMW(loggingMW(s.log)(recoverMW(s.log)(timeoutMW(s.timeout)(mux))))
}
type resolveRequest struct {
Address string `json:"address"`
// City is an optional hint for clients that keep the city separate from
// the street line. It overrides a city found in the address text.
City string `json:"city"`
}
func (s *Server) handleResolve(w http.ResponseWriter, r *http.Request) {
@@ -72,64 +79,49 @@ func (s *Server) handleResolve(w http.ResponseWriter, r *http.Request) {
if !ok {
return
}
writeJSON(w, http.StatusOK, s.resolver.Resolve(r.Context(), req.Address))
writeJSON(w, http.StatusOK, s.resolver.ResolveInCity(r.Context(), req.Address, req.City))
}
type outagesResponse struct {
Status address.Status `json:"status"`
Address string `json:"address,omitempty"`
District string `json:"district,omitempty"`
Outages []outages.Outage `json:"outages"`
AsOf string `json:"as_of,omitempty"`
DataVersion string `json:"data_version,omitempty"`
Detail string `json:"detail,omitempty"`
type cityRequest struct {
Query string `json:"query"`
}
func (s *Server) handleOutages(w http.ResponseWriter, r *http.Request) {
req, ok := s.decode(w, r)
if !ok {
return
}
ctx := r.Context()
res := s.resolver.Resolve(ctx, req.Address)
type cityResponse struct {
Status string `json:"status"`
City string `json:"city,omitempty"`
}
out := outagesResponse{
Status: res.Status,
Address: res.Address,
District: res.District,
DataVersion: res.DataVersion,
Outages: []outages.Outage{},
}
if res.Status != address.StatusFound {
// No canonical address => nothing to query. Return the resolver's
// verdict so the caller can ask for clarification.
writeJSON(w, http.StatusOK, out)
// handleCity normalizes a city-only query ("нск", "г. Новосибирск",
// "Новосибирская область"). It is the small, separate use case for projects
// that need city names but not streets.
func (s *Server) handleCity(w http.ResponseWriter, r *http.Request) {
var req cityRequest
r.Body = http.MaxBytesReader(w, r.Body, 16*1024)
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
if s.outages == nil {
// The events module is optional and off when portal.url is empty.
out.Status = address.StatusInfra
out.Detail = "outages source is not configured"
writeJSON(w, http.StatusOK, out)
if req.Query == "" {
writeError(w, http.StatusBadRequest, "query is required")
return
}
if utf8.RuneCountInString(req.Query) > maxCityQueryRunes {
writeError(w, http.StatusBadRequest, "query is too long")
return
}
if city, ok := s.resolver.MatchCity(req.Query); ok {
writeJSON(w, http.StatusOK, cityResponse{Status: "found", City: city})
return
}
writeJSON(w, http.StatusOK, cityResponse{Status: "not_found"})
}
raw, err := s.outages.Query(ctx, res.Address)
if err != nil {
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
out.Status = address.StatusPortalTimeout
} else {
out.Status = address.StatusInfra
}
writeJSON(w, http.StatusOK, out)
return
func (s *Server) handleCities(w http.ResponseWriter, _ *http.Request) {
cities := s.resolver.Cities()
if cities == nil {
cities = []string{}
}
parsed := address.Parse(res.Address)
current := outages.Current(raw, s.now(), s.loc)
out.Outages = outages.MatchingHouse(current, parsed)
out.AsOf = s.now().In(s.loc).Format(time.RFC3339)
writeJSON(w, http.StatusOK, out)
writeJSON(w, http.StatusOK, map[string][]string{"cities": cities})
}
func (s *Server) handleHealthz(w http.ResponseWriter, _ *http.Request) {
@@ -160,6 +152,14 @@ func (s *Server) decode(w http.ResponseWriter, r *http.Request) (resolveRequest,
writeError(w, http.StatusBadRequest, "address is required")
return resolveRequest{}, false
}
if utf8.RuneCountInString(req.Address) > maxAddressRunes {
writeError(w, http.StatusBadRequest, "address is too long")
return resolveRequest{}, false
}
if utf8.RuneCountInString(req.City) > 128 {
writeError(w, http.StatusBadRequest, "city is too long")
return resolveRequest{}, false
}
return req, true
}
internal/adapters/httpapi/server_test.go
+137 -48
View File
@@ -2,46 +2,51 @@ package httpapi
import (
"bytes"
"context"
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
"git.totmin.ru/en2zmax/addrkit/internal/domain/outages"
)
func entry(t *testing.T, city, raw string) address.Entry {
t.Helper()
p := address.NewParser(address.NewDictionary(cityDictionary()), "").ParseInCity(raw, city)
return address.Entry{
ID: raw, Address: raw, District: "Тестовый",
City: p.City, CityKey: p.CityKey,
Ordinal: p.Ordinal, NameKey: p.NameKey, NameDisplay: p.NameDisplay,
StreetType: p.StreetType, StreetKey: p.StreetKey,
House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter,
Building: p.Building, BuildingFrac: p.BuildingFrac, BuildingLetter: p.BuildingLetter,
Structure: p.Structure,
}
}
func cityDictionary() []address.City {
return []address.City{
{Name: "Новосибирск", Aliases: []string{"нск", "новосиб", "новосибирская область"}},
{Name: "Бердск"},
}
}
func testResolver(t *testing.T) *match.Resolver {
t.Helper()
var entries []address.Entry
for _, a := range []string{"Станционная улица 1/1", "2-я Станционная улица 1/1", "Искитимская улица 5"} {
p := address.Parse(a)
entries = append(entries, address.Entry{
ID: a, Address: a, District: "Тестовый",
Ordinal: p.Ordinal, NameKey: p.NameKey, StreetType: p.StreetType, StreetKey: p.StreetKey,
House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter,
})
entries = append(entries, entry(t, "", a))
}
return match.NewResolver(match.NewIndex("test-version", entries, match.DefaultOptions()), nil)
}
type fakeSource struct {
list []outages.Outage
err error
}
func (f fakeSource) Query(context.Context, string) ([]outages.Outage, error) {
return f.list, f.err
}
func newServer(t *testing.T, src fakeSource) *httptest.Server {
func newServer(t *testing.T, resolver Resolver) *httptest.Server {
t.Helper()
srv := New(testResolver(t), src, slog.New(slog.NewTextHandler(io.Discard, nil)), time.UTC)
srv := New(resolver, slog.New(slog.NewTextHandler(io.Discard, nil)))
ts := httptest.NewServer(srv.Handler())
t.Cleanup(ts.Close)
return ts
@@ -57,7 +62,7 @@ func postJSON(t *testing.T, url, body string) *http.Response {
}
func TestResolveEndpoint(t *testing.T) {
ts := newServer(t, fakeSource{})
ts := newServer(t, testResolver(t))
resp := postJSON(t, ts.URL+"/v1/resolve", `{"address":"2-я Станционная 1/1"}`)
defer resp.Body.Close()
@@ -73,38 +78,120 @@ func TestResolveEndpoint(t *testing.T) {
}
}
func TestOutagesEndpointFiltersHouse(t *testing.T) {
src := fakeSource{list: []outages.Outage{
{Address: "Станционная улица 1/1", System: "Теплоснабжение"},
{Address: "Станционная улица 1/2", System: "Теплоснабжение"},
}}
ts := newServer(t, src)
resp := postJSON(t, ts.URL+"/v1/outages", `{"address":"станционная 1/1"}`)
defer resp.Body.Close()
var out outagesResponse
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
t.Fatalf("decode: %v", err)
}
if out.Status != address.StatusFound {
t.Fatalf("status = %q, want found", out.Status)
}
if len(out.Outages) != 1 || out.Outages[0].Address != "Станционная улица 1/1" {
t.Fatalf("house filter failed: %+v", out.Outages)
func TestResolveValidation(t *testing.T) {
ts := newServer(t, testResolver(t))
for _, body := range []string{`{}`, `{"address":""}`, `not json`} {
resp := postJSON(t, ts.URL+"/v1/resolve", body)
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("body %q: status = %d, want 400", body, resp.StatusCode)
}
}
}
func TestResolveValidation(t *testing.T) {
ts := newServer(t, fakeSource{})
resp := postJSON(t, ts.URL+"/v1/resolve", `{}`)
defer resp.Body.Close()
func TestResolveRejectsOversizedAddress(t *testing.T) {
ts := newServer(t, testResolver(t))
body, _ := json.Marshal(map[string]string{"address": strings.Repeat("а", maxAddressRunes+1)})
resp := postJSON(t, ts.URL+"/v1/resolve", string(body))
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("status = %d, want 400", resp.StatusCode)
}
}
func TestResolveCityHint(t *testing.T) {
entries := []address.Entry{
entry(t, "Новосибирск", "Ленина улица 5"),
entry(t, "Бердск", "Ленина улица 5"),
}
resolver := match.NewResolver(
match.NewIndex("test", entries, match.DefaultOptions()),
nil,
)
ts := newServer(t, resolver)
resp := postJSON(t, ts.URL+"/v1/resolve", `{"address":"Ленина 5","city":"Бердск"}`)
defer resp.Body.Close()
var res match.Result
if err := json.NewDecoder(resp.Body).Decode(&res); err != nil {
t.Fatalf("decode: %v", err)
}
if res.Status != address.StatusFound || res.City != "Бердск" || res.Address != "Ленина улица 5" {
t.Fatalf("unexpected result: %+v", res)
}
}
func TestCitiesEndpoint(t *testing.T) {
entries := []address.Entry{
entry(t, "Новосибирск", "Ленина улица 5"),
entry(t, "Бердск", "Ленина улица 5"),
}
resolver := match.NewResolver(match.NewIndex("test", entries, match.DefaultOptions()), nil)
ts := newServer(t, resolver)
resp, err := http.Get(ts.URL + "/v1/cities")
if err != nil {
t.Fatalf("get: %v", err)
}
defer resp.Body.Close()
var body struct {
Cities []string `json:"cities"`
}
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
t.Fatalf("decode: %v", err)
}
if len(body.Cities) != 2 || body.Cities[0] != "Бердск" || body.Cities[1] != "Новосибирск" {
t.Fatalf("cities = %v", body.Cities)
}
}
func TestCityEndpoint(t *testing.T) {
parser := address.NewParser(address.NewDictionary(cityDictionary()), "")
resolver := match.NewResolverWithOptions(
match.NewIndex("test", nil, match.DefaultOptions()),
nil,
match.ResolverOptions{Parser: parser},
)
ts := newServer(t, resolver)
for _, tt := range []struct {
query string
city string
}{
{"нск", "Новосибирск"},
{"г. Новосибирск", "Новосибирск"},
{"Новосибирская область", "Новосибирск"},
{"Бердск", "Бердск"},
{"Томск", ""},
} {
body, _ := json.Marshal(map[string]string{"query": tt.query})
resp := postJSON(t, ts.URL+"/v1/city", string(body))
var got cityResponse
if err := json.NewDecoder(resp.Body).Decode(&got); err != nil {
resp.Body.Close()
t.Fatalf("%q: decode: %v", tt.query, err)
}
resp.Body.Close()
if tt.city == "" {
if got.Status != "not_found" {
t.Errorf("%q: status = %q, want not_found", tt.query, got.Status)
}
continue
}
if got.Status != "found" || got.City != tt.city {
t.Errorf("%q: %+v", tt.query, got)
}
}
resp := postJSON(t, ts.URL+"/v1/city", `{}`)
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("empty query status = %d, want 400", resp.StatusCode)
}
}
func TestReadyzAndVersion(t *testing.T) {
ts := newServer(t, fakeSource{})
ts := newServer(t, testResolver(t))
resp, err := http.Get(ts.URL + "/v1/readyz")
if err != nil {
@@ -120,9 +207,11 @@ func TestReadyzAndVersion(t *testing.T) {
t.Fatalf("version: %v", err)
}
defer resp.Body.Close()
var v map[string]string
_ = json.NewDecoder(resp.Body).Decode(&v)
if v["data_version"] != "test-version" {
t.Fatalf("version = %q", v["data_version"])
var body map[string]string
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
t.Fatalf("decode: %v", err)
}
if body["data_version"] != "test-version" {
t.Fatalf("data_version = %q", body["data_version"])
}
}
internal/adapters/llm/rewriter.go
+14 -3
View File
@@ -14,10 +14,12 @@ import (
_ "embed"
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"time"
"unicode/utf8"
"git.totmin.ru/en2zmax/addrkit/internal/config"
)
@@ -128,7 +130,9 @@ func (r *Rewriter) Rewrite(ctx context.Context, raw string) ([]string, error) {
}
var cr chatResponse
if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil {
// Bound the reply: a broken or hostile model must not be able to stream an
// unbounded JSON body into the service.
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&cr); err != nil {
return nil, fmt.Errorf("decode llm response: %w", err)
}
if len(cr.Choices) == 0 {
@@ -154,8 +158,15 @@ func parseVariants(content string) []string {
}
out := make([]string, 0, len(parsed.Variants))
for _, v := range parsed.Variants {
if s := strings.TrimSpace(v); s != "" {
out = append(out, s)
s := strings.TrimSpace(v)
// A variant is a short address; the cap also keeps a hostile model
// from making the resolver scan a megabyte-long token.
if s == "" || utf8.RuneCountInString(s) > 512 {
continue
}
out = append(out, s)
if len(out) == 3 {
break
}
}
return out
internal/adapters/llm/rewriter_test.go
+14
View File
@@ -86,3 +86,17 @@ func TestParseVariants(t *testing.T) {
})
}
}
func TestParseVariantsCapsCountAndLength(t *testing.T) {
long := strings.Repeat("x", 600)
content := `{"variants":["` + long + `","a","b","c","d"]}`
out := parseVariants(content)
if len(out) != 3 {
t.Fatalf("variants = %v, want 3", out)
}
for _, v := range out {
if len([]rune(v)) > 512 {
t.Fatalf("oversized variant accepted: %d runes", len([]rune(v)))
}
}
}
internal/adapters/registrystore/sqlite.go
+112 -43
View File
@@ -5,37 +5,49 @@ package registrystore
import (
"database/sql"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
_ "modernc.org/sqlite" // registers the "sqlite" driver
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
)
// SchemaVersion identifies the on-disk format. It is independent of the data
// version (the registry snapshot date) and must be bumped whenever the key
// format or columns change. A service refuses to open a database it does not
// understand instead of silently resolving against half-empty fields.
const SchemaVersion = "2"
const schema = `
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS address (
id TEXT PRIMARY KEY,
address TEXT NOT NULL,
district TEXT NOT NULL DEFAULT '',
ordinal TEXT NOT NULL DEFAULT '',
name_key TEXT NOT NULL DEFAULT '',
name_display TEXT NOT NULL DEFAULT '',
street_type TEXT NOT NULL DEFAULT '',
street_key TEXT NOT NULL,
house TEXT NOT NULL DEFAULT '',
house_frac TEXT NOT NULL DEFAULT '',
house_letter TEXT NOT NULL DEFAULT '',
building TEXT NOT NULL DEFAULT '',
building_frac TEXT NOT NULL DEFAULT '',
id TEXT PRIMARY KEY,
address TEXT NOT NULL,
district TEXT NOT NULL DEFAULT '',
city TEXT NOT NULL DEFAULT '',
city_key TEXT NOT NULL DEFAULT '',
ordinal TEXT NOT NULL DEFAULT '',
name_key TEXT NOT NULL DEFAULT '',
name_display TEXT NOT NULL DEFAULT '',
street_type TEXT NOT NULL DEFAULT '',
street_key TEXT NOT NULL,
house TEXT NOT NULL DEFAULT '',
house_frac TEXT NOT NULL DEFAULT '',
house_letter TEXT NOT NULL DEFAULT '',
building TEXT NOT NULL DEFAULT '',
building_frac TEXT NOT NULL DEFAULT '',
building_letter TEXT NOT NULL DEFAULT '',
structure TEXT NOT NULL DEFAULT ''
structure TEXT NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_address_street ON address(street_key);
CREATE INDEX IF NOT EXISTS idx_address_city ON address(city_key);
`
// Save writes entries to path atomically: it builds a temporary file and
@@ -44,13 +56,21 @@ CREATE INDEX IF NOT EXISTS idx_address_street ON address(street_key);
// The build uses the rollback journal (not WAL) on purpose, so that the result
// is a single file with no companion -wal/-shm that a rename would orphan.
func Save(path, version string, entries []address.Entry) error {
tmp := path + ".tmp"
removeDBFiles(tmp)
if len(entries) == 0 {
return errors.New("refusing to publish an empty index")
}
db, err := sql.Open("sqlite", "file:"+tmp+
"?_pragma=journal_mode(DELETE)"+
"&_pragma=synchronous(FULL)"+
"&_pragma=busy_timeout(5000)")
tmp := path + ".tmp"
// A previous failed build may have left a complete temp database with the
// same meta rows; reusing it would hit a primary-key conflict.
_ = os.Remove(tmp)
removeSidecars(tmp)
uri, err := dsn(tmp, false)
if err != nil {
return err
}
db, err := sql.Open("sqlite", uri)
if err != nil {
return fmt.Errorf("open temp db: %w", err)
}
@@ -68,12 +88,46 @@ func Save(path, version string, entries []address.Entry) error {
if err := os.Rename(tmp, path); err != nil {
return fmt.Errorf("publish db: %w", err)
}
// Drop only the temp file's journal siblings; the published base file must
// survive.
// The rename is atomic, but a crash immediately after it may lose the
// directory entry. Syncing the parent makes the publish durable.
syncDir(filepath.Dir(path))
removeSidecars(tmp)
return nil
}
// dsn builds a SQLite URI. Concatenating "file:" + path breaks on paths that
// contain "?", "#" or "%", so let net/url escape the path for us.
func dsn(path string, readOnly bool) (string, error) {
abs, err := filepath.Abs(path)
if err != nil {
return "", fmt.Errorf("resolve db path: %w", err)
}
u := url.URL{Scheme: "file", Path: abs}
q := u.Query()
q.Add("_pragma", "busy_timeout(5000)")
if readOnly {
q.Set("mode", "ro")
q.Add("_pragma", "query_only(1)")
} else {
q.Add("_pragma", "journal_mode(DELETE)")
q.Add("_pragma", "synchronous(FULL)")
}
u.RawQuery = q.Encode()
return u.String(), nil
}
// syncDir fsyncs a directory so a freshly renamed file survives a crash. It is
// best effort: some filesystems do not support directory fsync, and failing to
// sync here is no worse than not trying.
func syncDir(dir string) {
f, err := os.Open(dir)
if err != nil {
return
}
defer f.Close()
_ = f.Sync()
}
func insertAll(db *sql.DB, version string, entries []address.Entry) error {
tx, err := db.Begin()
if err != nil {
@@ -84,11 +138,15 @@ func insertAll(db *sql.DB, version string, entries []address.Entry) error {
if _, err := tx.Exec(`INSERT INTO meta(key, value) VALUES ('version', ?)`, version); err != nil {
return fmt.Errorf("insert meta: %w", err)
}
if _, err := tx.Exec(`INSERT INTO meta(key, value) VALUES ('schema_version', ?)`, SchemaVersion); err != nil {
return fmt.Errorf("insert schema version: %w", err)
}
stmt, err := tx.Prepare(`INSERT INTO address (
id, address, district, ordinal, name_key, name_display, street_type, street_key,
house, house_frac, house_letter, building, building_frac, building_letter, structure
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
id, address, district, city, city_key, ordinal, name_key, name_display,
street_type, street_key, house, house_frac, house_letter, building,
building_frac, building_letter, structure
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
if err != nil {
return fmt.Errorf("prepare insert: %w", err)
}
@@ -101,8 +159,9 @@ func insertAll(db *sql.DB, version string, entries []address.Entry) error {
id = fmt.Sprintf("row-%d", i)
}
if _, err := stmt.Exec(
id, e.Address, e.District, e.Ordinal, e.NameKey, e.NameDisplay, e.StreetType, e.StreetKey,
e.House, e.HouseFrac, e.HouseLetter, e.Building, e.BuildingFrac, e.BuildingLetter, e.Structure,
id, e.Address, e.District, e.City, e.CityKey, e.Ordinal, e.NameKey, e.NameDisplay,
e.StreetType, e.StreetKey, e.House, e.HouseFrac, e.HouseLetter, e.Building,
e.BuildingFrac, e.BuildingLetter, e.Structure,
); err != nil {
return fmt.Errorf("insert %q: %w", e.Address, err)
}
@@ -114,26 +173,41 @@ func insertAll(db *sql.DB, version string, entries []address.Entry) error {
}
// Load opens path read-only and returns the data version and all entries.
// It fails closed when the file was produced by an incompatible schema.
func Load(path string) (version string, entries []address.Entry, err error) {
if _, err := os.Stat(path); err != nil {
return "", nil, fmt.Errorf("stat db: %w", err)
}
db, err := sql.Open("sqlite", "file:"+path+
"?mode=ro"+
"&_pragma=query_only(1)"+
"&_pragma=busy_timeout(5000)")
uri, err := dsn(path, true)
if err != nil {
return "", nil, err
}
db, err := sql.Open("sqlite", uri)
if err != nil {
return "", nil, fmt.Errorf("open db: %w", err)
}
defer db.Close()
var schemaVersion string
if err := db.QueryRow(`SELECT value FROM meta WHERE key='schema_version'`).Scan(&schemaVersion); err != nil {
return "", nil, fmt.Errorf("read schema version (re-run addrkit seed): %w", err)
}
if schemaVersion != SchemaVersion {
return "", nil, fmt.Errorf(
"unsupported index schema %q (want %q); re-run addrkit seed",
schemaVersion, SchemaVersion,
)
}
if err := db.QueryRow(`SELECT value FROM meta WHERE key='version'`).Scan(&version); err != nil {
return "", nil, fmt.Errorf("read version: %w", err)
}
rows, err := db.Query(`SELECT id, address, district, ordinal, name_key, name_display,
street_type, street_key, house, house_frac, house_letter, building, building_frac,
building_letter, structure FROM address`)
rows, err := db.Query(`SELECT id, address, district, city, city_key, ordinal,
name_key, name_display, street_type, street_key, house, house_frac,
house_letter, building, building_frac, building_letter, structure
FROM address
ORDER BY city_key, street_key, house, house_frac, house_letter,
building, building_frac, building_letter, structure, address, id`)
if err != nil {
return "", nil, fmt.Errorf("query: %w", err)
}
@@ -142,9 +216,10 @@ func Load(path string) (version string, entries []address.Entry, err error) {
for rows.Next() {
var e address.Entry
if err := rows.Scan(
&e.ID, &e.Address, &e.District, &e.Ordinal, &e.NameKey, &e.NameDisplay,
&e.StreetType, &e.StreetKey, &e.House, &e.HouseFrac, &e.HouseLetter,
&e.Building, &e.BuildingFrac, &e.BuildingLetter, &e.Structure,
&e.ID, &e.Address, &e.District, &e.City, &e.CityKey, &e.Ordinal,
&e.NameKey, &e.NameDisplay, &e.StreetType, &e.StreetKey, &e.House,
&e.HouseFrac, &e.HouseLetter, &e.Building, &e.BuildingFrac,
&e.BuildingLetter, &e.Structure,
); err != nil {
return "", nil, fmt.Errorf("scan: %w", err)
}
@@ -156,12 +231,6 @@ func Load(path string) (version string, entries []address.Entry, err error) {
return version, entries, nil
}
// removeDBFiles deletes a database file and its journal siblings.
func removeDBFiles(base string) {
_ = os.Remove(base)
removeSidecars(base)
}
// removeSidecars deletes only the -wal/-shm companions of a database file.
func removeSidecars(base string) {
_ = os.Remove(base + "-wal")
internal/adapters/registrystore/sqlite_test.go
+58 -2
View File
@@ -1,7 +1,9 @@
package registrystore
import (
"database/sql"
"path/filepath"
"strings"
"testing"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
@@ -11,11 +13,13 @@ func TestSaveLoadRoundTrip(t *testing.T) {
path := filepath.Join(t.TempDir(), "registry.db")
addr := "Станционная улица 1/1"
p := address.Parse(addr)
parser := address.NewParser(address.NewDictionary([]address.City{{Name: "Новосибирск"}}), "")
p := parser.ParseInCity(addr, "Новосибирск")
entries := []address.Entry{{
ID: "a1", Address: addr, District: "Ленинский",
City: p.City, CityKey: p.CityKey,
Ordinal: p.Ordinal, NameKey: p.NameKey, StreetType: p.StreetType, StreetKey: p.StreetKey,
House: p.House, HouseFrac: p.HouseFrac,
House: p.House, HouseFrac: p.HouseFrac, Structure: "2",
}}
if err := Save(path, "2026-10-09", entries); err != nil {
@@ -35,6 +39,58 @@ func TestSaveLoadRoundTrip(t *testing.T) {
if got[0].Address != addr || got[0].District != "Ленинский" || got[0].StreetKey != p.StreetKey {
t.Errorf("round-tripped entry mismatch: %+v", got[0])
}
if got[0].City != "Новосибирск" || got[0].CityKey != "новосибирск" || got[0].Structure != "2" {
t.Errorf("city/structure not round-tripped: %+v", got[0])
}
}
func TestSaveRejectsEmptyIndex(t *testing.T) {
path := filepath.Join(t.TempDir(), "registry.db")
if err := Save(path, "v1", nil); err == nil {
t.Fatal("expected error for empty index")
}
}
func TestLoadRejectsIncompatibleSchema(t *testing.T) {
path := filepath.Join(t.TempDir(), "old.db")
uri, err := dsn(path, false)
if err != nil {
t.Fatalf("dsn: %v", err)
}
db, err := sql.Open("sqlite", uri)
if err != nil {
t.Fatalf("open: %v", err)
}
if _, err := db.Exec(`CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`); err != nil {
t.Fatalf("create meta: %v", err)
}
if _, err := db.Exec(`INSERT INTO meta(key, value) VALUES ('schema_version', '1')`); err != nil {
t.Fatalf("insert schema version: %v", err)
}
db.Close()
_, _, err = Load(path)
if err == nil || !strings.Contains(err.Error(), "unsupported index schema") {
t.Fatalf("Load error = %v, want unsupported schema", err)
}
}
// TestPathWithSpecialCharacters guards the SQLite URI construction: a naive
// "file:" + path breaks on ?, # and % in the path.
func TestPathWithSpecialCharacters(t *testing.T) {
path := filepath.Join(t.TempDir(), "registry ? # 100%.db")
entries := []address.Entry{{ID: "x", Address: "Первая улица 1"}}
if err := Save(path, "v1", entries); err != nil {
t.Fatalf("Save: %v", err)
}
version, got, err := Load(path)
if err != nil {
t.Fatalf("Load: %v", err)
}
if version != "v1" || len(got) != 1 {
t.Fatalf("version=%q entries=%+v", version, got)
}
}
// TestSaveIsAtomicOverwrite verifies that re-seeding replaces the previous data
internal/config/config.go
+87 -20
View File
@@ -4,11 +4,15 @@
package config
import (
"bytes"
"encoding/json"
"fmt"
"os"
"regexp"
"strings"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
)
// Config is the whole service configuration.
@@ -21,25 +25,24 @@ type Config struct {
LogLevel string `json:"log_level"`
Resolve ResolveConfig `json:"resolve"`
Portal PortalConfig `json:"portal"`
LLM LLMConfig `json:"llm"`
// Cities is the dictionary of cities this deployment understands. A
// single-city deployment lists one city and may set DefaultCity so that
// queries like "ленина 5" work without naming it.
Cities []address.City `json:"cities"`
DefaultCity string `json:"default_city"`
LLM LLMConfig `json:"llm"`
}
// ResolveConfig tunes the matching engine.
// ResolveConfig tunes the matching engine. Defaults come from
// match.DefaultOptions so there is exactly one source of truth.
type ResolveConfig struct {
FuzzyThreshold float64 `json:"fuzzy_threshold"`
AmbiguityMargin float64 `json:"ambiguity_margin"`
MaxCandidates int `json:"max_candidates"`
}
// PortalConfig configures the external outages portal (ArcGIS). When URL is
// empty the "events by house" module is disabled and only /v1/resolve works.
type PortalConfig struct {
URL string `json:"url"`
TimeoutSeconds int `json:"timeout_seconds"`
Retries int `json:"retries"`
}
// LLMConfig configures the optional, disabled-by-default query rewriter.
type LLMConfig struct {
Enabled bool `json:"enabled"`
@@ -63,17 +66,14 @@ type LLMConfig struct {
// Default returns a config with sane defaults for the non-required fields.
func Default() Config {
resolve := match.DefaultOptions()
return Config{
Addr: ":6171",
LogLevel: "info",
Resolve: ResolveConfig{
FuzzyThreshold: 0.82,
AmbiguityMargin: 0.03,
MaxCandidates: 5,
},
Portal: PortalConfig{
TimeoutSeconds: 2,
Retries: 1,
FuzzyThreshold: resolve.FuzzyThreshold,
AmbiguityMargin: resolve.AmbiguityMargin,
MaxCandidates: resolve.MaxCandidates,
},
LLM: LLMConfig{
TimeoutSeconds: 3,
@@ -91,6 +91,9 @@ func Load(path string) (Config, error) {
}
cfg := Default()
dec := json.NewDecoder(strings.NewReader(expandEnv(string(raw))))
// A typo in a key (fuzzy_treshold, data_pth, portal, ...) must not silently
// fall back to a default. Fail closed and let the operator fix the config.
dec.DisallowUnknownFields()
if err := dec.Decode(&cfg); err != nil {
return Config{}, fmt.Errorf("parse config: %w", err)
}
@@ -107,6 +110,25 @@ func Load(path string) (Config, error) {
return cfg, nil
}
// LoadCitiesFile reads a standalone JSON array of cities. It is used by the
// seed CLI so that seeding and serving share the same city vocabulary.
func LoadCitiesFile(path string) ([]address.City, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read cities: %w", err)
}
var cities []address.City
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&cities); err != nil {
return nil, fmt.Errorf("parse cities: %w", err)
}
if err := validateCities(cities, ""); err != nil {
return nil, err
}
return cities, nil
}
func (c Config) validate() error {
if c.Addr == "" {
return fmt.Errorf("config: addr is required")
@@ -114,11 +136,56 @@ func (c Config) validate() error {
if c.DataPath == "" {
return fmt.Errorf("config: data_path is required")
}
if c.Resolve.FuzzyThreshold <= 0 || c.Resolve.FuzzyThreshold > 1 {
return fmt.Errorf("config: resolve.fuzzy_threshold must be in (0, 1]")
}
if c.Resolve.AmbiguityMargin < 0 || c.Resolve.AmbiguityMargin > 1 {
return fmt.Errorf("config: resolve.ambiguity_margin must be in [0, 1]")
}
if c.Resolve.MaxCandidates < 1 {
return fmt.Errorf("config: resolve.max_candidates must be at least 1")
}
if c.LLM.Enabled && c.LLM.URL == "" {
return fmt.Errorf("config: llm.url is required when llm.enabled is true")
}
// portal.url is optional: without it the "events by house" module is
// disabled and only /v1/resolve is served.
return validateCities(c.Cities, c.DefaultCity)
}
// validateCities checks the city dictionary for the mistakes that would make
// matching silently ambiguous: empty names, duplicate cities and an alias
// claimed by two different cities.
func validateCities(cities []address.City, defaultCity string) error {
seenNames := make(map[string]string)
seenAliases := make(map[string]string)
for _, city := range cities {
name := strings.TrimSpace(city.Name)
key := address.NormalizeText(name)
if key == "" {
return fmt.Errorf("config: cities[].name must not be empty")
}
if prev, ok := seenNames[key]; ok {
return fmt.Errorf("config: duplicate city %q and %q", prev, name)
}
seenNames[key] = name
aliases := append([]string{name}, city.Aliases...)
for _, alias := range aliases {
norm := address.NormalizeText(alias)
if norm == "" {
continue
}
if owner, ok := seenAliases[norm]; ok && owner != key {
return fmt.Errorf("config: alias %q is claimed by both %q and %q", alias, owner, key)
}
seenAliases[norm] = key
}
}
if defaultCity = strings.TrimSpace(defaultCity); defaultCity != "" && len(cities) > 0 {
if _, ok := seenNames[address.NormalizeText(defaultCity)]; !ok {
return fmt.Errorf("config: default_city %q is not listed in cities", defaultCity)
}
}
return nil
}
internal/config/config_test.go
+85 -5
View File
@@ -4,6 +4,8 @@ import (
"os"
"path/filepath"
"testing"
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
)
func writeFile(t *testing.T, dir, name, content string) string {
@@ -19,7 +21,6 @@ func baseConfig(promptFields string) string {
return `{
"addr": ":6171",
"data_path": "registry.db",
"portal": { "url": "http://portal" },
"llm": { "enabled": true, "url": "http://llm"` + promptFields + ` }
}`
}
@@ -60,11 +61,90 @@ func TestLoadMissingSystemPromptFileFailsClosed(t *testing.T) {
}
}
func TestPortalURLOptional(t *testing.T) {
func TestDefaultsMatchMatcher(t *testing.T) {
dir := t.TempDir()
// No portal.url: the events module is disabled, the core still loads.
path := writeFile(t, dir, "config.json", `{"addr": ":6171", "data_path": "x.db"}`)
if _, err := Load(path); err != nil {
t.Fatalf("portal.url must be optional: %v", err)
cfg, err := Load(path)
if err != nil {
t.Fatalf("Load: %v", err)
}
want := match.DefaultOptions()
if cfg.Resolve.FuzzyThreshold != want.FuzzyThreshold ||
cfg.Resolve.AmbiguityMargin != want.AmbiguityMargin ||
cfg.Resolve.MaxCandidates != want.MaxCandidates {
t.Fatalf("config defaults %+v do not match matcher defaults %+v", cfg.Resolve, want)
}
}
func TestRejectsBadResolveRanges(t *testing.T) {
dir := t.TempDir()
cases := map[string]string{
"threshold": `{"addr":":6171","data_path":"x.db","resolve":{"fuzzy_threshold":0}}`,
"margin": `{"addr":":6171","data_path":"x.db","resolve":{"ambiguity_margin":2}}`,
"candidates": `{"addr":":6171","data_path":"x.db","resolve":{"max_candidates":0}}`,
}
for name, body := range cases {
path := writeFile(t, dir, name+".json", body)
if _, err := Load(path); err == nil {
t.Errorf("%s: expected validation error", name)
}
}
}
func TestCitiesAndDefaultCity(t *testing.T) {
dir := t.TempDir()
path := writeFile(t, dir, "config.json", `{
"addr": ":6171", "data_path": "x.db",
"cities": [
{"name": "Новосибирск", "aliases": ["нск", "новосибирская область"]},
{"name": "Бердск"}
],
"default_city": "Новосибирск"
}`)
cfg, err := Load(path)
if err != nil {
t.Fatalf("Load: %v", err)
}
if len(cfg.Cities) != 2 || cfg.DefaultCity != "Новосибирск" {
t.Fatalf("cities = %+v, default = %q", cfg.Cities, cfg.DefaultCity)
}
}
func TestRejectsUnknownDefaultCity(t *testing.T) {
dir := t.TempDir()
path := writeFile(t, dir, "config.json", `{
"addr": ":6171", "data_path": "x.db",
"cities": [{"name": "Бердск"}],
"default_city": "Новосибирск"
}`)
if _, err := Load(path); err == nil {
t.Fatal("expected error for default_city absent from cities")
}
}
func TestLoadCitiesFile(t *testing.T) {
dir := t.TempDir()
path := writeFile(t, dir, "cities.json", `[
{"name": "Новосибирск", "aliases": ["нск"]},
{"name": "Бердск"}
]`)
cities, err := LoadCitiesFile(path)
if err != nil {
t.Fatalf("LoadCitiesFile: %v", err)
}
if len(cities) != 2 || cities[0].Name != "Новосибирск" {
t.Fatalf("cities = %+v", cities)
}
}
func TestRejectsUnknownConfigFields(t *testing.T) {
dir := t.TempDir()
// "portal" was removed with the events module; silently ignoring it would
// hide a stale deployment config.
path := writeFile(t, dir, "config.json",
`{"addr":":6171","data_path":"x.db","portal":{"url":"http://example"}}`)
if _, err := Load(path); err == nil {
t.Fatal("expected error for unknown config field")
}
}
internal/domain/address/city_test.go
+141
View File
@@ -0,0 +1,141 @@
package address
import "testing"
func testDictionary() *Dictionary {
return NewDictionary([]City{
{Name: "Новосибирск", Aliases: []string{"нск", "новосиб", "новосибирская область", "нсо"}},
{Name: "Бердск"},
})
}
func TestParseCityFromAliases(t *testing.T) {
parser := NewParser(testDictionary(), "")
tests := []struct {
raw string
city string
nameKey string
}{
{"нск, 2-я Станционная улица 1", "Новосибирск", "станционная"},
{"НСК Ленина 5", "Новосибирск", "ленина"},
{"Новосибирская область, Станционная 1", "Новосибирск", "станционная"},
{"Бердск, Ленина 5", "Бердск", "ленина"},
{"г. Новосибирск, ул. Ленина 5", "Новосибирск", "ленина"},
}
for _, tt := range tests {
got := parser.Parse(tt.raw)
if got.City != tt.city {
t.Errorf("%q: City = %q, want %q", tt.raw, got.City, tt.city)
}
if got.NameKey != tt.nameKey {
t.Errorf("%q: NameKey = %q, want %q", tt.raw, got.NameKey, tt.nameKey)
}
}
}
func TestParseCityIsPartOfStreetKey(t *testing.T) {
parser := NewParser(testDictionary(), "")
nsk := parser.Parse("Новосибирск, Ленина 5")
berdsk := parser.Parse("Бердск, Ленина 5")
if nsk.StreetKey == berdsk.StreetKey {
t.Fatalf("same street key for different cities: %q", nsk.StreetKey)
}
if nsk.CityKey != "новосибирск" || berdsk.CityKey != "бердск" {
t.Fatalf("city keys = %q / %q", nsk.CityKey, berdsk.CityKey)
}
}
func TestParseDefaultCity(t *testing.T) {
parser := NewParser(testDictionary(), "Новосибирск")
got := parser.Parse("Ленина 5")
if got.City != "Новосибирск" || got.CityKey != "новосибирск" {
t.Fatalf("City = %q / %q", got.City, got.CityKey)
}
// The default city is also recognised when the user spells it out, even
// if the dictionary contains only aliases.
explicit := NewParser(DefaultDictionary(), "Новосибирск").Parse("г. Новосибирск, Ленина 5")
if explicit.City != "Новосибирск" || explicit.NameKey != "ленина" {
t.Fatalf("explicit default city: %+v", explicit)
}
}
func TestParseInCityOverridesDetectedCity(t *testing.T) {
parser := NewParser(testDictionary(), "")
got := parser.ParseInCity("Бердск, Ленина 5", "Новосибирск")
if got.City != "Новосибирск" {
t.Fatalf("City = %q, want forced Новосибирск", got.City)
}
if got.NameKey != "ленина" {
t.Fatalf("NameKey = %q", got.NameKey)
}
}
func TestParseWithoutCityKeepsCityKeyEmpty(t *testing.T) {
parser := NewParser(testDictionary(), "")
got := parser.Parse("Ленина 5")
if got.City != "" || got.CityKey != "" {
t.Fatalf("City = %q / %q, want empty", got.City, got.CityKey)
}
}
func TestNormalizeTextStillLeavesUnknownCityTokens(t *testing.T) {
// A city that is not in the dictionary must not be silently dropped: the
// resolver should fail rather than guess another city.
parser := NewParser(testDictionary(), "")
got := parser.Parse("Томск, Ленина 5")
if got.City != "" {
t.Fatalf("City = %q, want empty for unknown city", got.City)
}
if got.NameKey == "ленина" {
t.Fatalf("unknown city token disappeared from the street name")
}
}
func TestMatchCity(t *testing.T) {
dict := testDictionary()
cases := []struct {
query string
want string
ok bool
}{
{"нск", "Новосибирск", true},
{"г. Новосибирск", "Новосибирск", true},
{"Новосибирская область", "Новосибирск", true},
{"Бердск", "Бердск", true},
{"Бердск, Ленина 5", "", false},
{"Томск", "", false},
}
for _, tt := range cases {
got, ok := dict.MatchCity(tt.query)
if ok != tt.ok || got.Name != tt.want {
t.Errorf("MatchCity(%q) = %+v, %v; want %q, %v", tt.query, got, ok, tt.want, tt.ok)
}
}
}
func FuzzParse(f *testing.F) {
for _, seed := range []string{
"",
"2-я Станционная улица 1/1 стр.",
"нск, Ленина 5",
"Бердск, Ленина 5 к2",
"г. Новосибирск, ул. Кошурникова, д. 11/1 кв. 5",
"\x00\xff",
} {
f.Add(seed)
}
parser := NewParser(testDictionary(), "")
f.Fuzz(func(t *testing.T, raw string) {
first := parser.Parse(raw)
second := parser.Parse(raw)
if first.StreetKey != second.StreetKey {
t.Fatalf("non-deterministic street key for %q: %q vs %q", raw, first.StreetKey, second.StreetKey)
}
if NormalizeText(raw) != NormalizeText(NormalizeText(raw)) {
t.Fatalf("NormalizeText is not idempotent for %q", raw)
}
})
}
internal/domain/address/dictionary.go
+284
View File
@@ -0,0 +1,284 @@
package address
import (
"sort"
"strings"
)
// City is one entry of the configurable city dictionary.
//
// Aliases are additional surface forms that refer to the city: common
// abbreviations ("нск"), the region name ("новосибирская область") and so on.
// The canonical Name is always accepted as an alias too. Matching is
// token-based, case-insensitive and "ё"-insensitive, so aliases should be
// written naturally.
type City struct {
Name string `json:"name"`
Aliases []string `json:"aliases,omitempty"`
}
// Dictionary is the immutable, deployment-specific vocabulary of the parser:
// cities, filler words, street types and ordinals.
//
// Keeping this data in a value instead of package-level maps makes the parser
// reusable across deployments: a service in Novosibirsk and a service in Tomsk
// can run the same code with different city lists, without recompilation.
type Dictionary struct {
cities []cityAlias
filler map[string]struct{}
streetTypes map[string]string
ordinalWords map[string]string
ordinalSuffixes map[string]struct{}
}
// cityAlias is one token sequence that resolves to a city.
type cityAlias struct {
tokens []string
name string
key string
}
// NewDictionary builds a dictionary with the built-in federal vocabulary and
// the supplied cities. The input slice is copied; mutating it later does not
// affect the dictionary.
func NewDictionary(cities []City) *Dictionary {
d := &Dictionary{
filler: cloneSet(defaultFillerWords),
streetTypes: cloneMap(defaultStreetTypes),
ordinalWords: cloneMap(defaultOrdinalWords),
ordinalSuffixes: cloneSet(defaultOrdinalSuffixes),
}
seen := make(map[string]struct{})
for _, c := range cities {
name := strings.TrimSpace(c.Name)
cityKey := NormalizeText(name)
if name == "" || cityKey == "" {
continue
}
aliases := append([]string{name}, c.Aliases...)
for _, alias := range aliases {
tokens := strings.Fields(NormalizeText(alias))
if len(tokens) == 0 {
continue
}
dedup := cityKey + "|" + strings.Join(tokens, " ")
if _, ok := seen[dedup]; ok {
continue
}
seen[dedup] = struct{}{}
d.cities = append(d.cities, cityAlias{
tokens: tokens,
name: name,
key: cityKey,
})
}
}
// Longest aliases win, so "новосибирская область" is preferred over a
// hypothetical shorter "новосибирская". The key is the tie-breaker that
// keeps the order deterministic when two cities share an alias.
d.sortCities()
return d
}
// sortCities orders aliases by specificity: more tokens first, then longer
// text, then key. The order decides which city wins when aliases overlap.
func (d *Dictionary) sortCities() {
sort.SliceStable(d.cities, func(i, j int) bool {
a, b := d.cities[i], d.cities[j]
if len(a.tokens) != len(b.tokens) {
return len(a.tokens) > len(b.tokens)
}
as, bs := strings.Join(a.tokens, " "), strings.Join(b.tokens, " ")
if len(as) != len(bs) {
return len(as) > len(bs)
}
if a.key != b.key {
return a.key < b.key
}
return as < bs
})
}
// withDefaultCity returns a dictionary that also recognises name as a city.
// It lets a single-city deployment use "-city Новосибирск" (or default_city)
// without duplicating the canonical name in the aliases list.
func (d *Dictionary) withDefaultCity(name string) *Dictionary {
key := NormalizeText(name)
if key == "" {
return d
}
for _, a := range d.cities {
if a.key == key {
return d
}
}
out := *d
out.cities = append(append([]cityAlias(nil), d.cities...), cityAlias{
tokens: strings.Fields(key),
name: name,
key: key,
})
out.sortCities()
return &out
}
// DefaultDictionary returns the built-in vocabulary without cities. It is the
// parser used by address.Parse for callers that do not need city awareness.
func DefaultDictionary() *Dictionary {
return NewDictionary(nil)
}
// MatchCity reports whether query consists only of a city name/alias (plus
// generic filler such as "г.") and returns the canonical city. It is the
// city-only counterpart of Parse, useful for projects that normalize city
// names without a street.
func (d *Dictionary) MatchCity(query string) (City, bool) {
name, _, rest, ok := d.matchCity(strings.Fields(NormalizeText(query)))
if !ok {
return City{}, false
}
for _, tok := range rest {
if !d.isFiller(tok) {
return City{}, false
}
}
return City{Name: name}, true
}
// matchCity finds the longest city alias in tokens and returns the city plus
// the tokens with the alias removed. Matching is contiguous on token
// boundaries, so "нск" never matches inside another word.
func (d *Dictionary) matchCity(tokens []string) (name, key string, rest []string, ok bool) {
for _, alias := range d.cities {
if idx := indexTokens(tokens, alias.tokens); idx >= 0 {
rest = make([]string, 0, len(tokens)-len(alias.tokens))
rest = append(rest, tokens[:idx]...)
rest = append(rest, tokens[idx+len(alias.tokens):]...)
return alias.name, alias.key, rest, true
}
}
return "", "", tokens, false
}
// indexTokens returns the start of the first contiguous occurrence of want in
// tokens, or -1.
func indexTokens(tokens, want []string) int {
if len(want) == 0 || len(want) > len(tokens) {
return -1
}
for i := 0; i+len(want) <= len(tokens); i++ {
matched := true
for j := range want {
if tokens[i+j] != want[j] {
matched = false
break
}
}
if matched {
return i
}
}
return -1
}
func (d *Dictionary) isFiller(tok string) bool {
_, ok := d.filler[tok]
return ok
}
func (d *Dictionary) streetType(tok string) (string, bool) {
t, ok := d.streetTypes[tok]
return t, ok
}
func (d *Dictionary) ordinalWord(tok string) (string, bool) {
o, ok := d.ordinalWords[tok]
return o, ok
}
func (d *Dictionary) isOrdinalSuffix(tok string) bool {
_, ok := d.ordinalSuffixes[tok]
return ok
}
func cloneSet(src map[string]bool) map[string]struct{} {
dst := make(map[string]struct{}, len(src))
for k, v := range src {
if v {
dst[k] = struct{}{}
}
}
return dst
}
func cloneMap(src map[string]string) map[string]string {
dst := make(map[string]string, len(src))
for k, v := range src {
dst[k] = v
}
return dst
}
// defaultFillerWords are tokens that carry no street-name meaning and that
// people routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
//
// City names are deliberately NOT listed here: they come from Dictionary, so
// the same code can serve any city (or many cities) without edits.
var defaultFillerWords = map[string]bool{
"город": true, "г": true,
"область": true, "обл": true,
"район": true, "дом": true, "д": true,
}
// defaultStreetTypes maps a surface form (including common abbreviations) to a
// single canonical street type. The type is informational for matching (we
// strip it from the key) but useful for disambiguation and display.
var defaultStreetTypes = map[string]string{
"ул": "улица", "улица": "улица",
"пер": "переулок", "пер-к": "переулок", "переулок": "переулок",
"пр": "проезд", "проезд": "проезд",
"пр-т": "проспект", "просп": "проспект", "проспект": "проспект",
"ш": "шоссе", "шоссе": "шоссе",
"мкр": "микрорайон", "микрорайон": "микрорайон",
"туп": "тупик", "тупик": "тупик",
"спуск": "спуск",
"б-р": "бульвар", "бул": "бульвар", "бульвар": "бульвар",
"пл": "площадь", "площадь": "площадь",
"пос": "поселок", "поселок": "поселок",
"тракт": "тракт",
"аллея": "аллея",
"наб": "набережная", "набережная": "набережная",
"кольцо": "кольцо",
"просека": "просека",
"квартал": "квартал",
"территория": "территория",
"дорога": "дорога",
"разъезд": "разъезд",
}
// defaultOrdinalWords maps spelled-out ordinals to their digit form. Only the
// forms that actually occur in practice are listed; unknown words are treated
// as part of the street name.
var defaultOrdinalWords = map[string]string{
"первый": "1", "первая": "1", "первое": "1",
"второй": "2", "вторая": "2", "второе": "2",
"третий": "3", "третья": "3", "третье": "3",
"четвертый": "4", "четвертая": "4", "четвертое": "4",
"пятый": "5", "пятая": "5", "пятое": "5",
"шестой": "6", "шестая": "6", "шестое": "6",
"седьмой": "7", "седьмая": "7", "седьмое": "7",
"восьмой": "8", "восьмая": "8", "восьмое": "8",
"девятый": "9", "девятая": "9", "девятое": "9",
"десятый": "10", "десятая": "10", "десятое": "10",
}
// defaultOrdinalSuffixes are the gender/number endings that may follow an
// ordinal digit when it is written as a separate token ("2 я", "10 й").
var defaultOrdinalSuffixes = map[string]bool{
"я": true, "й": true, "е": true,
"ая": true, "ый": true, "ой": true, "ое": true,
"ий": true, "ья": true, "ье": true, "ьи": true,
}
internal/domain/address/normalize.go
+1 -1
View File
@@ -14,7 +14,7 @@ import (
var fold = cases.Fold()
// reTrailingParenthetical matches a trailing "(...)" group, e.g. the district
// suffix that the outages portal appends: "Медкадры улица, 1 (Заельцовский
// suffix that source systems append: "Медкадры улица, 1 (Заельцовский
// район)". Such administrative noise must not be treated as part of the house.
var reTrailingParenthetical = regexp.MustCompile(`\s*\([^)]*\)\s*$`)
internal/domain/address/parse.go
+113 -34
View File
@@ -24,42 +24,121 @@ var (
// Trailing apartment part: " кв 5", " квартира 12", " кв. 7а". Removed
// before house parsing so it cannot leak into the street name.
reApartment = regexp.MustCompile(`(?:^|\s)(?:квартира|кв)\.?(?:\s+\d+[а-я]?)?\s*$`)
// Spoken fractions: "18 дробь 1" is the same house unit as "18/1".
reFractionWord = regexp.MustCompile(`\s+(?:дробь|дробью)\s+`)
)
// Parse decomposes a free-form address into its structured components. It is
// total: unparseable input still yields a non-nil ParsedAddress whose street
// part is best-effort and whose HasHouse is false.
// Parser turns free-form address strings into structured addresses using a
// deployment-specific Dictionary. It is immutable and safe for concurrent use.
type Parser struct {
dict *Dictionary
defaultCityName string
defaultCityKey string
}
// NewParser builds a parser. defaultCity is used when the query does not name a
// city and the dictionary has no match; pass an empty string for deployments
// where the city must be present in every query.
func NewParser(dict *Dictionary, defaultCity string) *Parser {
if dict == nil {
dict = DefaultDictionary()
}
p := &Parser{}
if name := strings.TrimSpace(defaultCity); name != "" {
dict = dict.withDefaultCity(name)
p.defaultCityName = name
p.defaultCityKey = NormalizeText(name)
}
p.dict = dict
return p
}
// defaultParser is used by the package-level Parse convenience function. It has
// no cities and is therefore fully backward compatible: callers that need city
// awareness construct their own Parser.
var defaultParser = NewParser(DefaultDictionary(), "")
// DefaultParser returns the shared city-less parser.
func DefaultParser() *Parser {
return defaultParser
}
// Parse decomposes a free-form address with the default (city-less) parser.
func Parse(raw string) ParsedAddress {
return defaultParser.Parse(raw)
}
// Parse decomposes a free-form address into its structured components. It is
// total: unparseable input still yields a usable ParsedAddress whose street
// part is best-effort and whose HasHouse is false.
func (p *Parser) Parse(raw string) ParsedAddress {
return p.parse(raw, "")
}
// ParseInCity parses raw and forces cityName as the result's city. It is used
// by the seed pipeline, where the city is an explicit field of the source row
// and must win over whatever the address text contains.
func (p *Parser) ParseInCity(raw, cityName string) ParsedAddress {
return p.parse(raw, cityName)
}
// MatchCity reports whether query is a city name/alias and returns its
// canonical form. It does not guess a city from an arbitrary address.
func (p *Parser) MatchCity(query string) (City, bool) {
return p.dict.MatchCity(query)
}
func (p *Parser) parse(raw, forcedCity string) ParsedAddress {
normalized := NormalizeText(raw)
// A trailing apartment ("кв 5") is not part of the address identity.
normalized = strings.TrimSpace(reApartment.ReplaceAllString(normalized, ""))
p := ParsedAddress{Raw: raw, Normalized: normalized}
// Collapse the spoken fraction marker to the canonical separator before
// the right-to-left house parser runs: "18 дробь 1" -> "18/1".
normalized = reFractionWord.ReplaceAllString(normalized, "/")
parsed := ParsedAddress{Raw: raw, Normalized: normalized}
rest := normalized
if cityName, cityKey, restTokens, ok := p.dict.matchCity(strings.Fields(rest)); ok {
parsed.City = cityName
parsed.CityKey = cityKey
rest = strings.Join(restTokens, " ")
}
if forcedCity = strings.TrimSpace(forcedCity); forcedCity != "" {
parsed.City = forcedCity
parsed.CityKey = NormalizeText(forcedCity)
} else if parsed.City == "" && p.defaultCityName != "" {
// A single-city deployment can omit the city in queries. In a
// multi-city deployment default_city should stay empty: guessing a
// city is exactly the kind of silent error this service avoids.
parsed.City = p.defaultCityName
parsed.CityKey = p.defaultCityKey
}
if g, start, ok := matchSuffix(reStructure, rest); ok {
p.Structure = trimLeadingZeros(g[1])
parsed.Structure = trimLeadingZeros(g[1])
rest = strings.TrimSpace(rest[:start])
}
if g, start, ok := matchSuffix(reBuilding, rest); ok {
// The regex also matches a bare "к"; only accept it when a digit or a
// letter actually follows, otherwise it is not a corpus marker.
if g[1] != "" || g[2] != "" {
p.Building = trimLeadingZeros(g[1])
p.BuildingLetter = firstNonEmpty(g[2], g[4])
p.BuildingFrac = trimLeadingZeros(g[3])
parsed.Building = trimLeadingZeros(g[1])
parsed.BuildingLetter = firstNonEmpty(g[2], g[4])
parsed.BuildingFrac = trimLeadingZeros(g[3])
rest = strings.TrimSpace(rest[:start])
}
}
if g, start, ok := matchSuffix(reHouse, rest); ok {
p.House = trimLeadingZeros(g[1])
p.HouseLetter = firstNonEmpty(g[2], g[4])
p.HouseFrac = trimLeadingZeros(g[3])
p.HasHouse = true
parsed.House = trimLeadingZeros(g[1])
parsed.HouseLetter = firstNonEmpty(g[2], g[4])
parsed.HouseFrac = trimLeadingZeros(g[3])
parsed.HasHouse = true
rest = strings.TrimSpace(rest[:start])
}
parseStreet(&p, rest)
return p
p.parseStreet(&parsed, rest)
return parsed
}
// firstNonEmpty returns the first non-empty argument. House/building letters
@@ -73,8 +152,8 @@ func firstNonEmpty(values ...string) string {
return ""
}
// parseStreet fills the street-related fields of p from the leftover string
// (the address with its house block removed).
// parseStreet fills the street-related fields of out from the leftover string
// (the address with its city and house block removed).
//
// All recognised type tokens are removed from the name. This matters for
// registry rows such as "Гусинобродский Тракт улица 21", where both "Тракт"
@@ -82,7 +161,7 @@ func firstNonEmpty(values ...string) string {
// says "Гусинобродский 21". The first type token becomes the canonical type.
// If removing them would leave the name empty (a street literally called
// "Набережная"), the type token is kept as the name.
func parseStreet(p *ParsedAddress, s string) {
func (p *Parser) parseStreet(out *ParsedAddress, s string) {
tokens := strings.Fields(s)
name := make([]string, 0, len(tokens))
streetType := ""
@@ -90,10 +169,10 @@ func parseStreet(p *ParsedAddress, s string) {
for _, tok := range tokens {
// Filler tokens ("г", "город", "дом", "область", ...) never belong to
// the street name.
if fillerWords[tok] {
if p.dict.isFiller(tok) {
continue
}
if t, ok := streetTypes[tok]; ok {
if t, ok := p.dict.streetType(tok); ok {
if streetType == "" {
streetType = t
}
@@ -102,55 +181,55 @@ func parseStreet(p *ParsedAddress, s string) {
name = append(name, tok)
}
p.StreetType = streetType
out.StreetType = streetType
if len(name) == 0 && streetType != "" {
name = []string{streetType}
}
if len(name) == 0 {
p.StreetKey = streetKey("", "")
out.StreetKey = streetKey(out.CityKey, "", "")
return
}
name = extractOrdinal(p, name)
p.NameDisplay = strings.Join(name, " ")
name = p.extractOrdinal(out, name)
out.NameDisplay = strings.Join(name, " ")
sorted := append([]string(nil), name...)
sort.Strings(sorted)
p.NameKey = strings.Join(sorted, " ")
p.StreetKey = streetKey(p.Ordinal, p.NameKey)
out.NameKey = strings.Join(sorted, " ")
out.StreetKey = streetKey(out.CityKey, out.Ordinal, out.NameKey)
}
// extractOrdinal recognises an ordinal and removes it from the name tokens.
// It handles both the usual leading form ("2-я Сухарная") and an ordinal that
// appears after the name ("Сухарная 2-я улица"), which both people and models
// produce.
func extractOrdinal(p *ParsedAddress, tokens []string) []string {
func (p *Parser) extractOrdinal(out *ParsedAddress, tokens []string) []string {
if len(tokens) == 0 {
return tokens
}
// Leading forms first: they are by far the most common.
if m := reOrdinalToken.FindStringSubmatch(tokens[0]); m != nil {
p.Ordinal = normalizeOrdinal(m[1])
out.Ordinal = normalizeOrdinal(m[1])
return tokens[1:]
}
if isDigits(tokens[0]) && len(tokens) >= 2 && ordinalSuffixes[tokens[1]] {
p.Ordinal = normalizeOrdinal(tokens[0])
if isDigits(tokens[0]) && len(tokens) >= 2 && p.dict.isOrdinalSuffix(tokens[1]) {
out.Ordinal = normalizeOrdinal(tokens[0])
return tokens[2:]
}
if d, ok := ordinalWords[tokens[0]]; ok {
p.Ordinal = normalizeOrdinal(d)
if d, ok := p.dict.ordinalWord(tokens[0]); ok {
out.Ordinal = normalizeOrdinal(d)
return tokens[1:]
}
// Otherwise look for an ordinal anywhere among the name tokens.
for i, tok := range tokens {
if m := reOrdinalToken.FindStringSubmatch(tok); m != nil {
p.Ordinal = normalizeOrdinal(m[1])
out.Ordinal = normalizeOrdinal(m[1])
return removeRange(tokens, i, i+1)
}
if isDigits(tok) && i+1 < len(tokens) && ordinalSuffixes[tokens[i+1]] {
p.Ordinal = normalizeOrdinal(tok)
if isDigits(tok) && i+1 < len(tokens) && p.dict.isOrdinalSuffix(tokens[i+1]) {
out.Ordinal = normalizeOrdinal(tok)
return removeRange(tokens, i, i+2)
}
}
internal/domain/address/parse_test.go
+7 -1
View File
@@ -71,6 +71,12 @@ func TestParseHouseBlock(t *testing.T) {
wantName: "выборная", wantType: "улица",
wantHouse: "45", wantLetter: "а", wantFrac: "4", wantHas: true,
},
{
name: "spoken fraction word",
raw: "Горская улица 18 дробь 1",
wantName: "горская", wantType: "улица",
wantHouse: "18", wantFrac: "1", wantHas: true,
},
{
name: "corpus letter without digit",
raw: "1-я Ельцовка улица 1 кД",
@@ -96,7 +102,7 @@ func TestParseHouseBlock(t *testing.T) {
wantHouse: "11", wantFrac: "1", wantHas: true,
},
{
name: "portal district suffix in parentheses",
name: "district suffix in parentheses",
raw: "Медкадры улица, 1 (Заельцовский район)",
wantName: "медкадры", wantType: "улица",
wantHouse: "1", wantHas: true,
internal/domain/address/street.go
+13 -67
View File
@@ -2,73 +2,19 @@ package address
import "strings"
// fillerWords are tokens that carry no street-name meaning and that people
// routinely include ("г. Новосибирск", "улица ... дом 5", "... область").
// Dropping them makes the service tolerant of natural phrasing. District
// adjectives are deliberately NOT dropped: they can legitimately be part of a
// street name.
var fillerWords = map[string]bool{
"новосибирск": true, "нск": true, "нсо": true,
"город": true, "г": true,
"область": true, "обл": true,
"район": true, "дом": true, "д": true,
}
// streetTypes maps a surface form (including common abbreviations) to a single
// canonical street type. The type is informational for matching (we strip it
// from the key) but useful for disambiguation and display.
var streetTypes = map[string]string{
"ул": "улица", "улица": "улица",
"пер": "переулок", "пер-к": "переулок", "переулок": "переулок",
"пр": "проезд", "проезд": "проезд",
"пр-т": "проспект", "просп": "проспект", "проспект": "проспект",
"ш": "шоссе", "шоссе": "шоссе",
"мкр": "микрорайон", "микрорайон": "микрорайон",
"туп": "тупик", "тупик": "тупик",
"спуск": "спуск",
"б-р": "бульвар", "бул": "бульвар", "бульвар": "бульвар",
"пл": "площадь", "площадь": "площадь",
"пос": "поселок", "поселок": "поселок",
"тракт": "тракт",
"аллея": "аллея",
"наб": "набережная", "набережная": "набережная",
"кольцо": "кольцо",
"просека": "просека",
"квартал": "квартал",
"территория": "территория",
"дорога": "дорога",
"разъезд": "разъезд",
}
// ordinalWords maps spelled-out ordinals to their digit form. Only the forms
// that actually occur in practice are listed; unknown words are treated as
// part of the street name.
var ordinalWords = map[string]string{"первый": "1", "первая": "1", "первое": "1",
"второй": "2", "вторая": "2", "второе": "2",
"третий": "3", "третья": "3", "третье": "3",
"четвертый": "4", "четвертая": "4", "четвертое": "4",
"пятый": "5", "пятая": "5", "пятое": "5",
"шестой": "6", "шестая": "6", "шестое": "6",
"седьмой": "7", "седьмая": "7", "седьмое": "7",
"восьмой": "8", "восьмая": "8", "восьмое": "8",
"девятый": "9", "девятая": "9", "девятое": "9",
"десятый": "10", "десятая": "10", "десятое": "10",
}
// ordinalSuffixes are the gender/number endings that may follow an ordinal
// digit when it is written as a separate token ("2 я", "10 й").
var ordinalSuffixes = map[string]bool{
"я": true, "й": true, "е": true,
"ая": true, "ый": true, "ой": true, "ое": true,
"ий": true, "ья": true, "ье": true, "ьи": true,
}
// streetKey builds the canonical, type-independent street key:
// "<ordinal>|<sorted name tokens>". The type word is dropped so that a user
// who omits it ("станционная 1" instead of "Станционная улица 1") still
// matches; type confusion is surfaced later as ambiguity.
func streetKey(ordinal, nameKey string) string {
return ordinal + "|" + nameKey
// streetKey builds the canonical, city-aware, type-independent street key:
//
// "<cityKey>|<ordinal>|<sorted name tokens>"
//
// The city is part of the key so that "Ленина" in Novosibirsk and "Ленина" in
// Berdsk are different streets. The type word is dropped so that a user who
// omits it ("станционная 1" instead of "Станционная улица 1") still matches;
// type confusion is surfaced later as ambiguity.
//
// None of the components may contain "|": NormalizeText folds all punctuation
// that could appear in a name into spaces, and ordinals are digits only.
func streetKey(cityKey, ordinal, nameKey string) string {
return cityKey + "|" + ordinal + "|" + nameKey
}
// normalizeOrdinal trims leading zeros so "02-я" and "2-я" agree.
internal/domain/address/types.go
+11 -6
View File
@@ -10,7 +10,7 @@ package address
// Status is the machine-readable outcome of an address resolution.
//
// It is intentionally explicit: callers must never have to guess whether an
// empty result means "not found" or "the upstream failed".
// empty result means "not found" or "the resolver refused to choose".
type Status string
const (
@@ -25,11 +25,6 @@ const (
StatusAmbiguous Status = "ambiguous"
// StatusNotFound means nothing in the registry matched.
StatusNotFound Status = "not_found"
// StatusPortalTimeout means the external outages portal did not answer in
// time. It is an infrastructure outcome, not "no outages".
StatusPortalTimeout Status = "portal_timeout"
// StatusInfra is a generic terminal infrastructure failure.
StatusInfra Status = "infra_error"
)
// ParsedAddress is the structured form of a free-form address string.
@@ -41,6 +36,13 @@ type ParsedAddress struct {
Raw string `json:"raw"`
Normalized string `json:"normalized"`
// City is the canonical city name when the query names one (or when the
// parser is configured with a default city). CityKey is its normalized
// form. The city is part of StreetKey, so identical street names in
// different cities never collide.
City string `json:"city,omitempty"`
CityKey string `json:"city_key,omitempty"`
// Street components.
//
// NameKey is the order-insensitive canonical name (tokens sorted), so
@@ -73,6 +75,9 @@ type Entry struct {
Address string `json:"address"`
District string `json:"district,omitempty"`
City string `json:"city,omitempty"`
CityKey string `json:"city_key,omitempty"`
Ordinal string `json:"ordinal,omitempty"`
NameKey string `json:"name_key,omitempty"`
NameDisplay string `json:"name_display,omitempty"`
internal/domain/match/city_test.go
+198
View File
@@ -0,0 +1,198 @@
package match
import (
"context"
"reflect"
"strings"
"testing"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
)
// staticRewriter returns a fixed list of variants, emulating the LLM adapter.
type staticRewriter []string
func (s staticRewriter) Rewrite(context.Context, string) ([]string, error) { return s, nil }
// TestRewriteInheritsDetectedCity guards the multi-city LLM path: the embedded
// prompt tells the model to strip administrative words, so a correct variant
// usually has no city. It must inherit the city detected in the original query,
// while a variant naming a different city must still be rejected.
func TestRewriteInheritsDetectedCity(t *testing.T) {
parser := address.NewParser(address.NewDictionary(matchCityDictionary()), "")
entries := cityEntries(t, parser,
[2]string{"Новосибирск", "Кошурникова улица 5"},
[2]string{"Бердск", "Кошурникова улица 5"},
)
// A threshold above any typo score forces the deterministic path to miss.
ix := NewIndex("test", entries, Options{FuzzyThreshold: 0.999, MaxCandidates: 5})
cityless := NewResolverWithOptions(ix, staticRewriter{"Кошурникова 5"}, ResolverOptions{Parser: parser})
res := cityless.ResolveInCity(context.Background(), "нск кошурникиова 5", "")
if res.Status != address.StatusFound || res.City != "Новосибирск" {
t.Fatalf("cityless variant: %+v", res)
}
conflict := NewResolverWithOptions(ix, staticRewriter{"Бердск, Кошурникова 5"}, ResolverOptions{Parser: parser})
res = conflict.ResolveInCity(context.Background(), "нск кошурникиова 5", "")
if res.Status == address.StatusFound {
t.Fatalf("conflicting city variant was accepted: %+v", res)
}
}
func matchCityDictionary() []address.City {
return []address.City{
{Name: "Новосибирск", Aliases: []string{"нск"}},
{Name: "Бердск"},
}
}
func cityEntries(t *testing.T, parser *address.Parser, rows ...[2]string) []address.Entry {
t.Helper()
entries := make([]address.Entry, 0, len(rows))
for _, row := range rows {
p := parser.ParseInCity(row[1], row[0])
entries = append(entries, address.Entry{
ID: row[0] + "/" + row[1], Address: row[1],
City: p.City, CityKey: p.CityKey,
Ordinal: p.Ordinal, NameKey: p.NameKey, NameDisplay: p.NameDisplay,
StreetType: p.StreetType, StreetKey: p.StreetKey,
House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter,
Building: p.Building, BuildingFrac: p.BuildingFrac, BuildingLetter: p.BuildingLetter,
Structure: p.Structure,
})
}
return entries
}
func TestIndexKeepsCitiesApart(t *testing.T) {
parser := address.NewParser(address.NewDictionary(matchCityDictionary()), "")
entries := cityEntries(t, parser,
[2]string{"Новосибирск", "Ленина улица 5"},
[2]string{"Бердск", "Ленина улица 5"},
)
resolver := NewResolverWithOptions(
NewIndex("test", entries, DefaultOptions()),
nil,
ResolverOptions{Parser: parser},
)
nsk := resolver.ResolveInCity(t.Context(), "Ленина 5", "Новосибирск")
if nsk.Status != address.StatusFound || nsk.City != "Новосибирск" {
t.Fatalf("Новосибирск: %+v", nsk)
}
berdsk := resolver.ResolveInCity(t.Context(), "Ленина 5", "Бердск")
if berdsk.Status != address.StatusFound || berdsk.City != "Бердск" {
t.Fatalf("Бердск: %+v", berdsk)
}
// Without a city the resolver must not pick one: both streets are exact
// matches in different cities.
res := resolver.Resolve(t.Context(), "Ленина 5")
if res.Status != address.StatusAmbiguous {
t.Fatalf("cityless status = %q, want ambiguous (candidates: %+v)", res.Status, res.Candidates)
}
if len(res.Candidates) != 2 {
t.Fatalf("candidates = %+v, want one per city", res.Candidates)
}
if res.City != "" {
t.Fatalf("ambiguous cityless result must not report a city, got %q", res.City)
}
}
func TestCitylessQueryReportsTheOnlyCity(t *testing.T) {
parser := address.NewParser(address.NewDictionary(matchCityDictionary()), "")
entries := cityEntries(t, parser, [2]string{"Новосибирск", "Ленина улица 5"})
resolver := NewResolverWithOptions(
NewIndex("test", entries, DefaultOptions()),
nil,
ResolverOptions{Parser: parser},
)
res := resolver.Resolve(t.Context(), "Ленина 5")
if res.Status != address.StatusFound || res.City != "Новосибирск" {
t.Fatalf("result = %+v", res)
}
}
func TestStructureIsPartOfHouseIdentity(t *testing.T) {
ix := buildIndex(t,
"Станционная улица 1",
"Станционная улица 1 стр. 2",
)
found := ix.Resolve(address.Parse("станционная 1 стр. 2"))
if found.Status != address.StatusFound || found.Address != "Станционная улица 1 стр. 2" {
t.Fatalf("exact structure: %+v", found)
}
// A structure that does not exist must never resolve to another building.
wrong := ix.Resolve(address.Parse("станционная 1 стр. 3"))
if wrong.Status == address.StatusFound {
t.Fatalf("wrong structure resolved to %q", wrong.Address)
}
if wrong.Status != address.StatusAmbiguous {
t.Fatalf("wrong structure status = %q, want ambiguous", wrong.Status)
}
base := ix.Resolve(address.Parse("станционная 1"))
if base.Status != address.StatusFound || base.Address != "Станционная улица 1" {
t.Fatalf("base house: %+v", base)
}
}
func TestIndexOrderIsDeterministic(t *testing.T) {
addresses := []string{
"Станционная улица 1/1",
"Станционная улица 2/1",
"Станционная улица 3/1 к1",
"Станционная улица 3/1 к2",
"Искитимская улица 5",
}
forward := buildEntries(t, addresses...)
backward := buildEntries(t, reverse(addresses)...)
a := NewIndex("v", forward, DefaultOptions())
b := NewIndex("v", backward, DefaultOptions())
for _, q := range []string{"станционная 3/1", "станционная 1/2", "искитимсая 5"} {
ra := a.Resolve(address.Parse(q))
rb := b.Resolve(address.Parse(q))
if !reflect.DeepEqual(ra, rb) {
t.Fatalf("%q: results differ\n a=%+v\n b=%+v", q, ra, rb)
}
}
}
func TestLongTokenDoesNotMatch(t *testing.T) {
ix := buildIndex(t, "Станционная улица 1")
res := ix.Resolve(address.Parse(strings.Repeat("а", 4000)))
if res.Status != address.StatusNotFound {
t.Fatalf("status = %q, want not_found", res.Status)
}
}
func buildEntries(t *testing.T, addresses ...string) []address.Entry {
t.Helper()
entries := make([]address.Entry, 0, len(addresses))
for _, a := range addresses {
p := address.Parse(a)
entries = append(entries, address.Entry{
ID: a, Address: a,
Ordinal: p.Ordinal, NameKey: p.NameKey, NameDisplay: p.NameDisplay,
StreetType: p.StreetType, StreetKey: p.StreetKey,
House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter,
Building: p.Building, BuildingFrac: p.BuildingFrac, BuildingLetter: p.BuildingLetter,
Structure: p.Structure,
})
}
return entries
}
func reverse(in []string) []string {
out := make([]string, len(in))
for i, s := range in {
out[len(in)-1-i] = s
}
return out
}
internal/domain/match/index.go
+137 -32
View File
@@ -9,6 +9,7 @@ package match
import (
"sort"
"unicode/utf8"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
)
@@ -16,16 +17,19 @@ import (
// Candidate is one plausible match, returned alongside a non-"found" result so
// the caller (or an operator) can see what the resolver was choosing between.
type Candidate struct {
Address string `json:"address"`
District string `json:"district,omitempty"`
House string `json:"house,omitempty"`
Score float64 `json:"score"`
Address string `json:"address"`
City string `json:"city,omitempty"`
District string `json:"district,omitempty"`
House string `json:"house,omitempty"`
Structure string `json:"structure,omitempty"`
Score float64 `json:"score"`
}
// Result is the outcome of a resolution.
type Result struct {
Status address.Status `json:"status"`
Address string `json:"address,omitempty"`
City string `json:"city,omitempty"`
District string `json:"district,omitempty"`
Confidence float64 `json:"confidence"`
Candidates []Candidate `json:"candidates,omitempty"`
@@ -59,10 +63,10 @@ func DefaultOptions() Options {
}
type streetRef struct {
key string
ordinal string
nameKey string
streetType string
key string
cityKey string
ordinal string
nameKey string
}
// Index is an immutable, in-memory registry index. It is safe for concurrent
@@ -72,26 +76,46 @@ type Index struct {
opts Options
byKey map[string][]address.Entry
streets []streetRef
cities map[string]string // cityKey -> canonical display name
size int
}
// maxFuzzyNameRunes bounds the fuzzy scan. No real street name is longer than
// this; without the bound a single huge token would make every Levenshtein
// comparison walk a very long string and turn a request into a CPU sink.
const maxFuzzyNameRunes = 96
// NewIndex builds an index from canonical entries. Entries sharing a StreetKey
// are grouped; distinct streets are collected for typo-tolerant matching.
//
// Entries are sorted first, so candidate order is deterministic regardless of
// the order in which the storage adapter returned them.
func NewIndex(version string, entries []address.Entry, opts Options) *Index {
sorted := append([]address.Entry(nil), entries...)
sort.SliceStable(sorted, func(i, j int) bool { return entryLess(sorted[i], sorted[j]) })
ix := &Index{
version: version,
opts: opts,
byKey: make(map[string][]address.Entry, len(entries)),
byKey: make(map[string][]address.Entry, len(sorted)),
cities: make(map[string]string),
size: len(sorted),
}
seen := make(map[string]struct{}, len(entries))
for _, e := range entries {
seen := make(map[string]struct{}, len(sorted))
for _, e := range sorted {
ix.byKey[e.StreetKey] = append(ix.byKey[e.StreetKey], e)
if e.CityKey != "" {
if _, ok := ix.cities[e.CityKey]; !ok {
ix.cities[e.CityKey] = e.City
}
}
if _, ok := seen[e.StreetKey]; !ok {
seen[e.StreetKey] = struct{}{}
ix.streets = append(ix.streets, streetRef{
key: e.StreetKey,
ordinal: e.Ordinal,
nameKey: e.NameKey,
streetType: e.StreetType,
key: e.StreetKey,
cityKey: e.CityKey,
ordinal: e.Ordinal,
nameKey: e.NameKey,
})
}
}
@@ -99,22 +123,57 @@ func NewIndex(version string, entries []address.Entry, opts Options) *Index {
return ix
}
// entryLess defines the canonical order of registry rows. Address strings are
// the last meaningful tie-breaker; ID makes the order total even for duplicated
// rows.
func entryLess(a, b address.Entry) bool {
switch {
case a.CityKey != b.CityKey:
return a.CityKey < b.CityKey
case a.StreetKey != b.StreetKey:
return a.StreetKey < b.StreetKey
case a.House != b.House:
return a.House < b.House
case a.HouseFrac != b.HouseFrac:
return a.HouseFrac < b.HouseFrac
case a.HouseLetter != b.HouseLetter:
return a.HouseLetter < b.HouseLetter
case a.Building != b.Building:
return a.Building < b.Building
case a.BuildingFrac != b.BuildingFrac:
return a.BuildingFrac < b.BuildingFrac
case a.BuildingLetter != b.BuildingLetter:
return a.BuildingLetter < b.BuildingLetter
case a.Structure != b.Structure:
return a.Structure < b.Structure
case a.Address != b.Address:
return a.Address < b.Address
default:
return a.ID < b.ID
}
}
// Version returns the data version the index was built from.
func (ix *Index) Version() string { return ix.version }
// Size returns the number of indexed entries.
func (ix *Index) Size() int {
n := 0
for _, v := range ix.byKey {
n += len(v)
func (ix *Index) Size() int { return ix.size }
// Cities returns the distinct canonical city names present in the index, in
// deterministic order. A city-less index returns an empty slice.
func (ix *Index) Cities() []string {
out := make([]string, 0, len(ix.cities))
for _, name := range ix.cities {
out = append(out, name)
}
return n
sort.Strings(out)
return out
}
// Resolve resolves a parsed address. It never returns an address that is not
// present in the index.
func (ix *Index) Resolve(p address.ParsedAddress) Result {
res := Result{DataVersion: ix.version}
res := Result{DataVersion: ix.version, City: p.City}
entries := ix.byKey[p.StreetKey]
similarity := 1.0
@@ -133,6 +192,11 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
res.Status = address.StatusNotFound
return res
}
if res.City == "" {
// A city-less query may still match exactly one city; report it so the
// caller does not have to re-derive the city from the candidates.
res.City = commonCity(entries)
}
if !p.HasHouse {
// Street is known but the caller did not supply a house: safe but
@@ -148,6 +212,7 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
case 1:
res.Status = address.StatusFound
res.Address = matched[0].Address
res.City = matched[0].City
res.District = matched[0].District
res.Confidence = clamp01(similarity)
return res
@@ -162,7 +227,14 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
}
res.Status = address.StatusPartial
res.Confidence = clamp01(similarity * 0.9)
res.Candidates = candidates(entries, similarity, ix.opts.MaxCandidates)
// Prefer houses that share the number/fraction/letter: they are the
// plausible corrections (a different corpus or structure), while the
// rest of the street is noise.
related := matchHouseUnitAll(entries, p)
if len(related) == 0 {
related = entries
}
res.Candidates = candidates(related, similarity, ix.opts.MaxCandidates)
return res
default:
res.Status = address.StatusAmbiguous
@@ -176,12 +248,16 @@ func (ix *Index) Resolve(p address.ParsedAddress) Result {
// the best street when the match is unambiguous, or the list of competing
// street keys when several are within the ambiguity margin.
func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry, similarity float64, ambiguousKeys []string) {
if p.NameKey == "" {
if p.NameKey == "" || utf8.RuneCountInString(p.NameKey) > maxFuzzyNameRunes {
return nil, 0, nil
}
best := 0.0
bestKey := ""
for _, s := range ix.streets {
// A query that names a city may only match streets in that city.
if p.CityKey != "" && s.cityKey != p.CityKey {
continue
}
// If the query carries an ordinal, the candidate's ordinal must match:
// a user who says "1-й Северный" does not mean the plain "Северная".
if !ordinalCompatible(p.Ordinal, s.ordinal) {
@@ -197,6 +273,9 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
}
for _, s := range ix.streets {
if p.CityKey != "" && s.cityKey != p.CityKey {
continue
}
if !ordinalCompatible(p.Ordinal, s.ordinal) {
continue
}
@@ -232,10 +311,12 @@ func (ix *Index) representatives(keys []string, score float64) []Candidate {
}
e := es[0]
out = append(out, Candidate{
Address: e.Address,
District: e.District,
House: e.House,
Score: clamp01(score),
Address: e.Address,
City: e.City,
District: e.District,
House: e.House,
Structure: e.Structure,
Score: clamp01(score),
})
if len(out) == ix.opts.MaxCandidates {
break
@@ -275,13 +356,16 @@ func sameHouseSeveral(entries []address.Entry, p address.ParsedAddress) bool {
// houseUnitEqual compares the parts of a house unit. Unspecified (empty) parts
// must also be empty on the entry side, so "60/1" never matches "60/1 к148".
// The structure ("стр. 2") is part of the identity: a caller who explicitly
// asked for a different structure must not be handed another building.
func houseUnitEqual(e address.Entry, p address.ParsedAddress) bool {
return e.House == p.House &&
e.HouseFrac == p.HouseFrac &&
e.HouseLetter == p.HouseLetter &&
e.Building == p.Building &&
e.BuildingFrac == p.BuildingFrac &&
e.BuildingLetter == p.BuildingLetter
e.BuildingLetter == p.BuildingLetter &&
e.Structure == p.Structure
}
func candidates(entries []address.Entry, score float64, limit int) []Candidate {
@@ -291,10 +375,12 @@ func candidates(entries []address.Entry, score float64, limit int) []Candidate {
out := make([]Candidate, 0, len(entries))
for _, e := range entries {
out = append(out, Candidate{
Address: e.Address,
District: e.District,
House: e.House,
Score: clamp01(score),
Address: e.Address,
City: e.City,
District: e.District,
House: e.House,
Structure: e.Structure,
Score: clamp01(score),
})
if len(out) == limit {
break
@@ -312,3 +398,22 @@ func clamp01(v float64) float64 {
}
return v
}
// commonCity returns the city shared by all entries, or "" when they span
// cities (or carry no city at all).
func commonCity(entries []address.Entry) string {
key, name := "", ""
for _, e := range entries {
if e.CityKey == "" {
continue
}
if key == "" {
key, name = e.CityKey, e.City
continue
}
if e.CityKey != key {
return ""
}
}
return name
}
internal/domain/match/levenshtein.go
+19 -4
View File
@@ -1,6 +1,9 @@
package match
import "strings"
import (
"strings"
"unicode/utf8"
)
// levenshtein returns the edit distance between two strings, measured in
// runes. It uses the rolling-row formulation: O(min(n,m)) memory.
@@ -102,18 +105,30 @@ func min3(a, b, c int) int {
return m
}
// maxTokenRunes is the longest token the fuzzy matcher will compare. Real
// street-name tokens are far shorter; anything longer is noise or abuse.
const maxTokenRunes = 64
// tokenScore scores two name tokens. A single edit is tolerated for tokens of
// length >= 4 and two edits for length >= 8; otherwise it falls back to the
// plain ratio. Short names (<= 3 chars) are intentionally not fuzzy-matched:
// one edit is too large a fraction of the word to be safe.
//
// Tokens longer than maxTokenRunes are never similar to a real street name.
// Rejecting them up front keeps a pathological input from turning one request
// into millions of Levenshtein cell updates.
func tokenScore(a, b string) float64 {
if a == b {
return 1
}
ra, rb := utf8.RuneCountInString(a), utf8.RuneCountInString(b)
if ra == 0 || rb == 0 || ra > maxTokenRunes || rb > maxTokenRunes {
return 0
}
d := levenshtein(a, b)
minLen := len([]rune(a))
if lb := len([]rune(b)); lb < minLen {
minLen = lb
minLen := ra
if rb < minLen {
minLen = rb
}
switch {
case d == 1 && minLen >= 4:
internal/domain/match/resolver.go
+102 -11
View File
@@ -20,6 +20,25 @@ type Rewriter interface {
Rewrite(ctx context.Context, raw string) ([]string, error)
}
// Parser is the address parser used by the resolver. It is defined here
// (consumer side) so the resolver can be tested with a stub and so a
// deployment can plug in its own dictionary.
type Parser interface {
Parse(raw string) address.ParsedAddress
ParseInCity(raw, city string) address.ParsedAddress
MatchCity(query string) (address.City, bool)
}
// ResolverOptions tunes optional resolver behaviour.
type ResolverOptions struct {
// Parser defaults to address.DefaultParser().
Parser Parser
// OnRewriteError, when set, observes failures of the optional rewriter.
// The resolver always keeps the deterministic result, but operators still
// need to know that the fallback is broken.
OnRewriteError func(error)
}
// Resolver wraps an Index with an optional Rewriter.
//
// The deterministic path is always tried first. The rewriter is consulted only
@@ -27,13 +46,30 @@ type Rewriter interface {
// suggestions must pass compatibleVariant before being considered, so the
// model cannot substitute a different street or corrupt the house unit.
type Resolver struct {
index *Index
rewriter Rewriter
index *Index
rewriter Rewriter
parser Parser
onRewriteError func(error)
}
// NewResolver builds a resolver. rewriter may be nil to disable rewriting.
// NewResolver builds a resolver with the default parser and no rewriter
// diagnostics.
func NewResolver(index *Index, rewriter Rewriter) *Resolver {
return &Resolver{index: index, rewriter: rewriter}
return NewResolverWithOptions(index, rewriter, ResolverOptions{})
}
// NewResolverWithOptions builds a resolver with explicit options.
func NewResolverWithOptions(index *Index, rewriter Rewriter, opts ResolverOptions) *Resolver {
parser := opts.Parser
if parser == nil {
parser = address.DefaultParser()
}
return &Resolver{
index: index,
rewriter: rewriter,
parser: parser,
onRewriteError: opts.OnRewriteError,
}
}
// Version returns the current data version.
@@ -42,9 +78,35 @@ func (r *Resolver) Version() string { return r.index.Version() }
// Ready reports whether the resolver has a usable index.
func (r *Resolver) Ready() bool { return r.index != nil && r.index.Size() > 0 }
// Cities returns the canonical city names present in the index.
func (r *Resolver) Cities() []string {
if r.index == nil {
return nil
}
return r.index.Cities()
}
// MatchCity reports whether query is a city name/alias and returns the
// canonical name. It uses the resolver's parser dictionary, not the index, so
// it also recognizes aliases for cities that have no rows yet.
func (r *Resolver) MatchCity(query string) (string, bool) {
city, ok := r.parser.MatchCity(query)
if !ok {
return "", false
}
return city.Name, true
}
// Resolve parses raw and resolves it, consulting the rewriter when enabled.
func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
parsed := address.Parse(raw)
return r.ResolveInCity(ctx, raw, "")
}
// ResolveInCity is Resolve with an explicit city hint. The hint overrides a
// city found in the address text and lets clients pass the city as a separate
// field.
func (r *Resolver) ResolveInCity(ctx context.Context, raw, city string) Result {
parsed := r.parse(raw, city)
res := r.index.Resolve(parsed)
if res.Status == address.StatusFound || r.rewriter == nil {
@@ -55,13 +117,27 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
}
alternatives, err := r.rewriter.Rewrite(ctx, raw)
if err != nil || len(alternatives) == 0 {
if err != nil {
if r.onRewriteError != nil {
r.onRewriteError(err)
}
return res
}
if len(alternatives) == 0 {
return res
}
best := res
for _, alt := range alternatives {
variant := address.Parse(alt)
variant := r.parse(alt, city)
// The embedded prompt asks the model to strip administrative words
// (including the city), so a correct variant often has no city. It
// inherits the original's city; only a conflicting explicit city is a
// disqualification. Without this, the rewriter silently contributes
// nothing in multi-city deployments (where default_city is empty).
if variant.CityKey == "" && parsed.CityKey != "" {
variant = r.parser.ParseInCity(alt, parsed.City)
}
// The model may fix the spelling of the street, nothing else.
if !compatibleVariant(parsed, variant) {
continue
@@ -74,6 +150,13 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
return best
}
func (r *Resolver) parse(raw, city string) address.ParsedAddress {
if strings.TrimSpace(city) != "" {
return r.parser.ParseInCity(raw, city)
}
return r.parser.Parse(raw)
}
// compatibleVariant reports whether an LLM rewrite may be considered. A rewrite
// is acceptable only when it:
// - keeps the exact house unit (number, fraction, letter, corpus, structure
@@ -82,6 +165,9 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
// - has the same words in the street name, each within a small edit distance
// — so a different street can never be substituted.
func compatibleVariant(orig, variant address.ParsedAddress) bool {
if orig.CityKey != variant.CityKey {
return false
}
if !sameHouseUnit(orig, variant) {
return false
}
@@ -108,9 +194,13 @@ func nearToken(a, b string) bool {
if a == b {
return true
}
minLen := len([]rune(a))
if lb := len([]rune(b)); lb < minLen {
minLen = lb
ra, rb := len([]rune(a)), len([]rune(b))
if ra == 0 || rb == 0 || ra > maxTokenRunes || rb > maxTokenRunes {
return false
}
minLen := ra
if rb < minLen {
minLen = rb
}
d := damerau(a, b)
switch {
@@ -129,7 +219,8 @@ func sameHouseUnit(a, b address.ParsedAddress) bool {
a.HouseLetter == b.HouseLetter &&
a.Building == b.Building &&
a.BuildingFrac == b.BuildingFrac &&
a.BuildingLetter == b.BuildingLetter
a.BuildingLetter == b.BuildingLetter &&
a.Structure == b.Structure
}
// better ranks results: a stronger status wins, then higher confidence.
internal/domain/match/resolver_test.go
+20
View File
@@ -42,6 +42,26 @@ func TestCompatibleVariantProtectsApartment(t *testing.T) {
}
}
// TestCompatibleVariantProtectsCityAndStructure makes sure a rewrite cannot
// move the address to another city or another building.
func TestCompatibleVariantProtectsCityAndStructure(t *testing.T) {
parser := address.NewParser(address.NewDictionary([]address.City{
{Name: "Новосибирск", Aliases: []string{"нск"}},
{Name: "Бердск"},
}), "")
orig := parser.Parse("нск, Сухарная улица 95 стр. 1")
if !compatibleVariant(orig, parser.Parse("Новосибирск, Сухраная улица 95 стр. 1")) {
t.Error("same-city spelling fix must be accepted")
}
if compatibleVariant(orig, parser.Parse("Бердск, Сухарная улица 95 стр. 1")) {
t.Error("city change must be rejected")
}
if compatibleVariant(orig, parser.Parse("нск, Сухарная улица 95 стр. 2")) {
t.Error("structure change must be rejected")
}
}
func TestDamerau(t *testing.T) {
if got := damerau("гоглоя", "гоголя"); got != 1 {
t.Errorf("transposition distance = %d, want 1", got)
internal/domain/outages/outages.go
-132
View File
@@ -1,132 +0,0 @@
// Package outages contains the deterministic, pure logic for selecting outages
// that belong to a concrete house unit. It does not talk to the network: the
// portal client lives in an adapter and feeds this package.
package outages
import (
"context"
"strconv"
"strings"
"time"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
)
// Outage is a single outage record as returned by the external portal.
type Outage struct {
Type string `json:"type,omitempty"`
System string `json:"system,omitempty"`
Address string `json:"address,omitempty"`
StartDate string `json:"start_date,omitempty"`
EndDate string `json:"end_date,omitempty"`
Time string `json:"time,omitempty"`
Action string `json:"action,omitempty"`
// Unit is the structured house unit, filled by the source adapter so the
// domain never has to parse the raw external address format. Not exported
// to JSON: it is an internal matching key.
Unit HouseUnit `json:"-"`
}
// HouseUnit identifies a concrete house within a street: number, optional
// fraction and letter, and the corpus (building) with its own fraction and
// letter. "Outages by house" matches on exactly this.
type HouseUnit struct {
House string
HouseFrac string
HouseLetter string
Building string
BuildingFrac string
BuildingLetter string
}
// HouseUnitOf extracts the unit from a parsed address.
func HouseUnitOf(p address.ParsedAddress) HouseUnit {
return HouseUnit{
House: p.House,
HouseFrac: p.HouseFrac,
HouseLetter: p.HouseLetter,
Building: p.Building,
BuildingFrac: p.BuildingFrac,
BuildingLetter: p.BuildingLetter,
}
}
// Source is anything that can answer "what outages are there for this
// canonical address". It is implemented by the ArcGIS portal adapter and by
// fakes in tests.
type Source interface {
Query(ctx context.Context, canonical string) ([]Outage, error)
}
// MatchingHouse keeps only the outages that address the same house unit as the
// target. This is the fix for the classic bug where a substring match returned
// a neighbouring house's outage: matching is done on the parsed structure, not
// on the raw text.
func MatchingHouse(list []Outage, target address.ParsedAddress) []Outage {
want := HouseUnitOf(target)
out := make([]Outage, 0, len(list))
for _, o := range list {
unit := o.Unit
if unit == (HouseUnit{}) {
// The source did not pre-parse the address (older/alternative
// adapters); fall back to parsing it here. Adapters should set
// Unit so the domain stays independent of external formats.
unit = HouseUnitOf(address.Parse(o.Address))
}
if unit == want {
out = append(out, o)
}
}
return out
}
// Current keeps outages whose [start, end] range covers now. The portal
// already filters by date, but we re-check locally because the portal's notion
// of "today" and ours must agree (Asia/Novosibirsk, UTC+7).
func Current(list []Outage, now time.Time, loc *time.Location) []Outage {
out := make([]Outage, 0, len(list))
for _, o := range list {
start, okStart := parsePortalTime(o.StartDate, loc)
end, okEnd := parsePortalTime(o.EndDate, loc)
if !okStart || !okEnd {
// Undated records are kept: better to report a possibly-current
// outage than to silently drop it.
out = append(out, o)
continue
}
if (now.Equal(start) || now.After(start)) && (now.Equal(end) || now.Before(end)) {
out = append(out, o)
}
}
return out
}
// parsePortalTime accepts the shapes the portal may produce: epoch
// milliseconds, epoch seconds, RFC3339 and a few common date formats. All
// results are interpreted in loc.
func parsePortalTime(s string, loc *time.Location) (time.Time, bool) {
s = strings.TrimSpace(s)
if s == "" {
return time.Time{}, false
}
if n, err := strconv.ParseInt(s, 10, 64); err == nil {
// Heuristic: 13+ digits is milliseconds, otherwise seconds.
if n > 1e12 {
return time.UnixMilli(n).In(loc), true
}
return time.Unix(n, 0).In(loc), true
}
layouts := []string{
time.RFC3339,
"2006-01-02T15:04:05",
"2006-01-02 15:04:05",
"2006-01-02",
}
for _, layout := range layouts {
if t, err := time.ParseInLocation(layout, s, loc); err == nil {
return t, true
}
}
return time.Time{}, false
}
internal/domain/outages/outages_test.go
-79
View File
@@ -1,79 +0,0 @@
package outages
import (
"strconv"
"testing"
"time"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
)
func TestMatchingHouse(t *testing.T) {
list := []Outage{
{Address: "Станционная улица 1/1", System: "Теплоснабжение"},
{Address: "Станционная улица 1/2", System: "Теплоснабжение"},
{Address: "Станционная улица 1/1 к1", System: "Теплоснабжение"},
{Address: "Станционная улица 10", System: "Теплоснабжение"},
}
target := address.Parse("2-я Станционная улица 1/1")
got := MatchingHouse(list, target)
if len(got) != 1 {
t.Fatalf("expected exactly 1 matching outage, got %d: %+v", len(got), got)
}
if got[0].Address != "Станционная улица 1/1" {
t.Errorf("matched wrong outage: %q", got[0].Address)
}
}
func TestMatchingHouseIgnoresStructure(t *testing.T) {
list := []Outage{{Address: "Станционная улица 1/1 стр.2"}}
target := address.Parse("станционная 1/1 стр.3")
if got := MatchingHouse(list, target); len(got) != 1 {
t.Fatalf("structure must not affect house matching; got %d", len(got))
}
}
// TestMatchingHouseDistrictSuffix covers the fallback path (Unit not set by the
// adapter): the parser must itself drop the portal's "(<district> район)".
func TestMatchingHouseDistrictSuffix(t *testing.T) {
list := []Outage{{Address: "Медкадры улица, 1 (Заельцовский район)"}}
target := address.Parse("Медкадры улица 1")
if got := MatchingHouse(list, target); len(got) != 1 {
t.Fatalf("district suffix must not break house matching; got %d", len(got))
}
}
// TestMatchingHouseUsesStructuredUnit verifies the domain matches on the unit
// the adapter parsed, not on the raw address text.
func TestMatchingHouseUsesStructuredUnit(t *testing.T) {
list := []Outage{{Address: "неважно", Unit: HouseUnit{House: "5", HouseFrac: "1"}}}
if got := MatchingHouse(list, address.Parse("что угодно 5/1")); len(got) != 1 {
t.Fatalf("structured unit not used for matching; got %d", len(got))
}
if got := MatchingHouse(list, address.Parse("что угодно 5/2")); len(got) != 0 {
t.Fatalf("different unit must not match; got %d", len(got))
}
}
func TestCurrent(t *testing.T) {
loc := time.FixedZone("NSK", 7*3600)
now := time.Date(2026, 10, 9, 12, 0, 0, 0, loc)
within := now.Add(-time.Hour).UnixMilli()
after := now.Add(time.Hour).UnixMilli()
before := now.Add(-48 * time.Hour).UnixMilli()
list := []Outage{
{Address: "A", StartDate: itoa(within), EndDate: itoa(after)}, // current
{Address: "B", StartDate: itoa(before), EndDate: itoa(before)}, // past
{Address: "C", StartDate: "", EndDate: ""}, // undated -> kept
}
got := Current(list, now, loc)
if len(got) != 2 {
t.Fatalf("expected 2 current outages, got %d: %+v", len(got), got)
}
}
func itoa(v int64) string {
return strconv.FormatInt(v, 10)
}
internal/eval/eval.go
+36 -10
View File
@@ -26,8 +26,12 @@ import (
// information (ordinal, house letter, fraction) do not require an exact match;
// for them a different, real address is "lossy" rather than "wrong".
type Case struct {
Query string
Expect string
Query string
Expect string
// ExpectCity is the canonical city of the source entry. Empty for a
// city-less index; otherwise a "found" result in another city is a safety
// violation even when the street and house text look identical.
ExpectCity string
Category string
RequireExact bool
}
@@ -41,6 +45,7 @@ type CategoryStat struct {
Ambiguous int `json:"ambiguous"`
NotFound int `json:"not_found"`
WrongHouse int `json:"wrong_house"`
WrongCity int `json:"wrong_city"`
WrongStreet int `json:"wrong_street"`
Lossy int `json:"lossy"`
}
@@ -72,11 +77,19 @@ func Run(resolver *match.Resolver, cases []Case) Report {
case address.StatusFound:
rp := address.Parse(res.Address)
equivalent := rp.StreetKey == exp.StreetKey && sameUnit(rp, exp)
sameCity := c.ExpectCity == "" || res.City == c.ExpectCity
switch {
case equivalent:
case equivalent && sameCity:
st.Exact++
rep.Exact++
exact = true
case !sameCity:
// A wrong city is always a safety violation, even when the
// case is allowed to be lossy: the address text can be
// identical across cities.
st.WrongCity++
rep.Wrong++
rep.WrongExamples = appendExample(rep.WrongExamples, c, res.Address)
case !c.RequireExact:
// Missing information resolved to a different real address:
// expected, not a safety violation.
@@ -111,14 +124,17 @@ func Run(resolver *match.Resolver, cases []Case) Report {
return rep
}
// sameUnit compares house units, ignoring the structure component.
// sameUnit compares house units, including the structure: two buildings that
// differ only by "стр." are different addresses.
func sameUnit(a, b address.ParsedAddress) bool {
return a.House == b.House &&
return a.CityKey == b.CityKey &&
a.House == b.House &&
a.HouseFrac == b.HouseFrac &&
a.HouseLetter == b.HouseLetter &&
a.Building == b.Building &&
a.BuildingFrac == b.BuildingFrac &&
a.BuildingLetter == b.BuildingLetter
a.BuildingLetter == b.BuildingLetter &&
a.Structure == b.Structure
}
func appendExample(examples []string, c Case, got string) []string {
@@ -185,16 +201,24 @@ func perturbations(e address.Entry, rng *rand.Rand) []Case {
var out []Case
add := func(query, category string, requireExact bool) {
if query != "" && query != base {
out = append(out, Case{Query: query, Expect: base, Category: category, RequireExact: requireExact})
out = append(out, Case{
Query: query, Expect: base, ExpectCity: e.City,
Category: category, RequireExact: requireExact,
})
}
}
// Baseline: the canonical address itself must always resolve exactly.
out = append(out, Case{Query: base, Expect: base, Category: "baseline", RequireExact: true})
out = append(out, Case{
Query: base, Expect: base, ExpectCity: e.City,
Category: "baseline", RequireExact: true,
})
add(strings.ToLower(base), "case_lower", true)
add(strings.ToUpper(base), "case_upper", true)
add("г. Новосибирск, "+base, "city_prefix", true)
if e.City != "" {
add("г. "+e.City+", "+base, "city_prefix", true)
}
add(insertDom(base, e.House), "filler_dom", true)
add(corruptPunctuation(base), "punctuation", true)
add(addYo(base), "yo", true)
@@ -212,7 +236,9 @@ func perturbations(e address.Entry, rng *rand.Rand) []Case {
add(strings.Replace(base, "к"+e.Building, "корпус "+e.Building, 1), "corpus_synonym", true)
}
if hasStructure(base) {
add(dropStructure(base), "structure_drop", true)
// Dropping "стр. N" discards information: the base house may or may
// not exist on its own, so a different real address is acceptable.
add(dropStructure(base), "structure_drop", false)
}
if q := mutateLongestToken(base, rng); q != "" {
add(q, "typo", true)
internal/eval/eval_test.go
+37
View File
@@ -4,6 +4,9 @@ import (
"math/rand"
"testing"
"unicode/utf8"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
)
// TestMoveTypeFront guards against the slice-aliasing regression that made the
@@ -47,3 +50,37 @@ func TestMutateOnlyNameTokens(t *testing.T) {
}
}
}
// TestRunDetectsWrongCity guards the multi-city safety net: canonical address
// text is identical across cities, so a found result must be checked against
// the expected city, not only against street/house text.
func TestRunDetectsWrongCity(t *testing.T) {
parser := address.NewParser(address.NewDictionary([]address.City{{Name: "Бердск"}}), "")
p := parser.ParseInCity("Ленина улица 5", "Бердск")
entries := []address.Entry{{
ID: "b1", Address: "Ленина улица 5",
City: p.City, CityKey: p.CityKey,
Ordinal: p.Ordinal, NameKey: p.NameKey, StreetKey: p.StreetKey, House: p.House,
}}
resolver := match.NewResolverWithOptions(
match.NewIndex("test", entries, match.DefaultOptions()),
nil,
match.ResolverOptions{Parser: parser},
)
cases := []Case{
{Query: "Ленина 5", Expect: "Ленина улица 5", ExpectCity: "Новосибирск", Category: "wrong_city", RequireExact: true},
{Query: "Ленина 5", Expect: "Ленина улица 5", ExpectCity: "Бердск", Category: "right_city", RequireExact: true},
}
rep := Run(resolver, cases)
if got := rep.Categories["wrong_city"]; got.WrongCity != 1 || got.Exact != 0 {
t.Fatalf("wrong_city = %+v, want WrongCity=1 Exact=0", got)
}
if got := rep.Categories["right_city"]; got.Exact != 1 || got.WrongCity != 0 {
t.Fatalf("right_city = %+v, want Exact=1 WrongCity=0", got)
}
if rep.Wrong != 1 {
t.Fatalf("report Wrong = %d, want 1", rep.Wrong)
}
}
internal/seed/realdata_test.go
+1 -1
View File
@@ -29,7 +29,7 @@ func TestRealRegistryParseRate(t *testing.T) {
if err != nil {
t.Fatalf("read registry: %v", err)
}
entries, stats := Build(sources)
entries, stats := Build(sources, BuildOptions{})
rate := 0.0
if stats.Total > 0 {
internal/seed/seed.go
+75 -19
View File
@@ -20,24 +20,38 @@ type Source struct {
ID string
Address string
District string
// City is optional: when the dump is already split per city it may be
// omitted and supplied as the default city to Build.
City string
}
// Stats summarises a build and, crucially, surfaces rows the parser could not
// structure. A non-empty UnparsedCount is a data-quality signal worth alerting
// on, not an error.
type Stats struct {
Total int `json:"total"`
Parsed int `json:"parsed"`
Unparsed int `json:"unparsed"`
Duplicates int `json:"duplicates"`
DistinctStreets int `json:"distinct_streets"`
UnparsedSample []string `json:"unparsed_sample,omitempty"`
Total int `json:"total"`
Parsed int `json:"parsed"`
Unparsed int `json:"unparsed"`
Duplicates int `json:"duplicates"`
DistinctStreets int `json:"distinct_streets"`
Cities map[string]int `json:"cities,omitempty"`
UnparsedSample []string `json:"unparsed_sample,omitempty"`
}
// BuildOptions controls how source rows become canonical entries.
type BuildOptions struct {
// Parser defaults to address.DefaultParser().
Parser *address.Parser
// DefaultCity is used for rows that neither carry a city field nor name a
// city in the address. Leave it empty for multi-city dumps where every row
// is expected to identify its city.
DefaultCity string
}
// ReadBulkNDJSON reads an Elasticsearch Bulk NDJSON dump: pairs of an action
// line ({"index":{"_id":...}}) followed by a source line
// ({"address":...,"district_name":...}). Bare source lines (no action line)
// are also accepted, which makes plain JSONL usable too.
// ({"address":...,"district_name":...,"city":...}). Bare source lines (no
// action line) are also accepted, which makes plain JSONL usable too.
func ReadBulkNDJSON(r io.Reader) ([]Source, error) {
sc := bufio.NewScanner(r)
// 1 MiB per line is far beyond any realistic address row.
@@ -75,6 +89,7 @@ func ReadBulkNDJSON(r io.Reader) ([]Source, error) {
if d, ok := obj["district_name"]; ok {
_ = json.Unmarshal(d, &src.District)
}
src.City = firstString(obj, "city", "city_name")
sources = append(sources, src)
pendingID = ""
}
@@ -85,17 +100,40 @@ func ReadBulkNDJSON(r io.Reader) ([]Source, error) {
return sources, nil
}
// firstString returns the first non-empty string field among names.
func firstString(obj map[string]json.RawMessage, names ...string) string {
for _, name := range names {
raw, ok := obj[name]
if !ok {
continue
}
var s string
if err := json.Unmarshal(raw, &s); err == nil && strings.TrimSpace(s) != "" {
return s
}
}
return ""
}
// Build parses every source row into a canonical Entry and reports statistics.
// Rows without a house number are still indexed for street lookups but counted
// as unparsed, since a registry row without a house is suspicious.
func Build(sources []Source) ([]address.Entry, Stats) {
func Build(sources []Source, opts BuildOptions) ([]address.Entry, Stats) {
parser := opts.Parser
if parser == nil {
parser = address.NewParser(address.DefaultDictionary(), opts.DefaultCity)
}
entries := make([]address.Entry, 0, len(sources))
streets := make(map[string]struct{})
seen := make(map[string]struct{})
stats := Stats{Total: len(sources)}
stats := Stats{Total: len(sources), Cities: make(map[string]int)}
for _, src := range sources {
p := address.Parse(src.Address)
// Precedence: an explicit source field wins over a city found in the
// address text; the parser's default city is the last resort. Passing
// an empty hint is what lets the parser apply that precedence.
cityHint := strings.TrimSpace(src.City)
p := parser.ParseInCity(src.Address, cityHint)
if p.NameKey == "" {
stats.Unparsed++
appendSample(&stats, src.Address)
@@ -110,9 +148,11 @@ func Build(sources []Source) ([]address.Entry, Stats) {
}
entry := address.Entry{
ID: src.ID,
ID: namespacedID(src.ID, p.CityKey),
Address: src.Address,
District: src.District,
City: p.City,
CityKey: p.CityKey,
Ordinal: p.Ordinal,
NameKey: p.NameKey,
NameDisplay: p.NameDisplay,
@@ -128,9 +168,8 @@ func Build(sources []Source) ([]address.Entry, Stats) {
}
// The source registry contains rows that differ only by "стр."
// (structure). Since structure does not affect which house an outage
// belongs to, drop such duplicates rather than surfacing them as
// spurious ambiguity at query time.
// (structure). They are distinct buildings, so structure is part of
// the de-duplication key and an exact structure query can be honoured.
if key := dedupKey(entry); key != "" {
if _, ok := seen[key]; ok {
stats.Duplicates++
@@ -141,24 +180,41 @@ func Build(sources []Source) ([]address.Entry, Stats) {
entries = append(entries, entry)
streets[p.StreetKey] = struct{}{}
stats.Cities[cityLabel(p.City)]++
}
stats.DistinctStreets = len(streets)
sort.Strings(stats.UnparsedSample)
return entries, stats
}
// dedupKey identifies the same house unit within the same street, ignoring the
// structure component.
// namespacedID keeps source ids unique when several city registries are merged
// into one database. Source systems often number houses independently.
func namespacedID(id, cityKey string) string {
if id == "" || cityKey == "" {
return id
}
return cityKey + "/" + id
}
// dedupKey identifies the same house unit within the same city and street,
// including its structure and building components.
func dedupKey(e address.Entry) string {
if e.House == "" {
return ""
}
return strings.Join([]string{
e.StreetKey, e.House, e.HouseFrac, e.HouseLetter,
e.Building, e.BuildingFrac, e.BuildingLetter,
e.CityKey, e.StreetKey, e.House, e.HouseFrac, e.HouseLetter,
e.Building, e.BuildingFrac, e.BuildingLetter, e.Structure,
}, "|")
}
func cityLabel(city string) string {
if city == "" {
return "(none)"
}
return city
}
func appendSample(s *Stats, address string) {
const maxSamples = 20
if len(s.UnparsedSample) < maxSamples {
internal/seed/seed_test.go
+73 -3
View File
@@ -3,11 +3,13 @@ package seed
import (
"strings"
"testing"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
)
func TestReadBulkNDJSON(t *testing.T) {
input := `{"index": {"_index": "address_registry", "_id": "a1"}}
{"address": "Станционная улица 1/1", "district_name": "Ленинский"}
{"address": "Станционная улица 1/1", "district_name": "Ленинский", "city": "Новосибирск"}
{"index": {"_id": "a2"}}
{"address": "Искитимская улица 5", "district_name": "Кировский"}
@@ -20,7 +22,7 @@ func TestReadBulkNDJSON(t *testing.T) {
if len(src) != 3 {
t.Fatalf("got %d sources, want 3", len(src))
}
if src[0].ID != "a1" || src[0].District != "Ленинский" {
if src[0].ID != "a1" || src[0].District != "Ленинский" || src[0].City != "Новосибирск" {
t.Errorf("first source = %+v", src[0])
}
if src[2].Address != "Третья улица 7" {
@@ -34,7 +36,7 @@ func TestBuild(t *testing.T) {
{ID: "2", Address: "Искитимская улица 5"},
{ID: "3", Address: "Станционная улица"}, // no house -> unparsed
}
entries, stats := Build(sources)
entries, stats := Build(sources, BuildOptions{})
if stats.Total != 3 {
t.Errorf("Total = %d, want 3", stats.Total)
@@ -53,3 +55,71 @@ func TestBuild(t *testing.T) {
t.Errorf("DistinctStreets = %d, want 2", stats.DistinctStreets)
}
}
func TestBuildKeepsCitiesApart(t *testing.T) {
sources := []Source{
{ID: "1", Address: "Ленина улица 5", City: "Новосибирск"},
{ID: "1", Address: "Ленина улица 5", City: "Бердск"},
{ID: "2", Address: "Ленина улица 7", City: "Новосибирск"},
}
entries, stats := Build(sources, BuildOptions{})
if len(entries) != 3 {
t.Fatalf("entries = %d, want 3: %+v", len(entries), entries)
}
if stats.DistinctStreets != 2 {
t.Errorf("DistinctStreets = %d, want 2 (same street in two cities)", stats.DistinctStreets)
}
if entries[0].StreetKey == entries[1].StreetKey {
t.Errorf("streets from different cities share a key: %q", entries[0].StreetKey)
}
if entries[0].ID == entries[1].ID {
t.Errorf("source ids collided across cities: %q", entries[0].ID)
}
if stats.Cities["Новосибирск"] != 2 || stats.Cities["Бердск"] != 1 {
t.Errorf("city stats = %v", stats.Cities)
}
}
func TestBuildAppliesDefaultCity(t *testing.T) {
sources := []Source{{ID: "1", Address: "Станционная улица 1/1"}}
entries, _ := Build(sources, BuildOptions{DefaultCity: "Новосибирск"})
if len(entries) != 1 {
t.Fatalf("entries = %d", len(entries))
}
if entries[0].City != "Новосибирск" || entries[0].CityKey != "новосибирск" {
t.Errorf("city = %q / %q", entries[0].City, entries[0].CityKey)
}
}
func TestBuildKeepsStructuresDistinct(t *testing.T) {
sources := []Source{
{ID: "1", Address: "Станционная улица 1"},
{ID: "2", Address: "Станционная улица 1 стр. 2"},
}
entries, stats := Build(sources, BuildOptions{})
if len(entries) != 2 {
t.Fatalf("entries = %d, want 2 (structure is part of identity)", len(entries))
}
if stats.Duplicates != 0 {
t.Errorf("Duplicates = %d, want 0", stats.Duplicates)
}
if entries[0].Structure == entries[1].Structure {
t.Errorf("structures collapsed: %+v", entries)
}
}
func TestBuildParsesCityFromAddress(t *testing.T) {
parser := address.NewParser(address.NewDictionary([]address.City{
{Name: "Новосибирск", Aliases: []string{"нск", "новосибирская область"}},
}), "")
sources := []Source{{ID: "1", Address: "нск, Станционная улица 1/1"}}
entries, _ := Build(sources, BuildOptions{Parser: parser})
if len(entries) != 1 {
t.Fatalf("entries = %d", len(entries))
}
if entries[0].City != "Новосибирск" || entries[0].NameKey != "станционная" {
t.Errorf("entry = %+v", entries[0])
}
}