en2zmax/addrkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor!: make addrkit a multi-city address normalizer

Browse Source 
- remove the events/outages module, ArcGIS adapter, portal config and event statuses
- add a configurable city dictionary/parser and make City part of StreetKey
- seed: read city/city_name, namespace IDs by city, keep structures distinct
- fix reliability: schema v2 with fail-closed load, deterministic ordering,
  input limits, config validation, atomic DB publish, LLM response caps
- add read-only shadow mode comparing the old ES path with addrkit
- breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
Maksim Totmin
2026-10-27 14:35:51 +07:00
parent a0b4cb2b14
commit 17ff5c591f
38 changed files with 2978 additions and 1164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/adapters/arcgis/client.go
-256
View File
@@ -1,256 +0,0 @@
// Package arcgis is the adapter for the external outages portal exposed as an
// ArcGIS MapServer /query endpoint.
//
// The portal only supports substring (LIKE) filtering, so precision is
// guaranteed locally: whatever the portal returns is re-filtered by the
// outages domain using the parsed house unit. The client adds the reliability
// the portal lacks: a hard timeout, bounded retries with backoff and context
// cancellation.
package arcgis
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
"git.totmin.ru/en2zmax/addrkit/internal/domain/outages"
)
// reDistrictSuffix matches the administrative suffix the portal appends to its
// display address, e.g. "Медкадры улица, 1 (Заельцовский район)". It must be
// removed before the address is parsed, otherwise the trailing district text
// hides the house number. The district is available separately as a field.
var reDistrictSuffix = regexp.MustCompile(`\s*\([^)]*\)\s*$`)
// cleanAddress strips the portal's district suffix and surrounding spaces.
func cleanAddress(s string) string {
return strings.TrimSpace(reDistrictSuffix.ReplaceAllString(s, ""))
}
// Client is a bounded HTTP client for the portal.
type Client struct {
baseURL string
http *http.Client
retries int
}
// New builds a client. timeout applies to a single attempt; retries is the
// number of *additional* attempts after the first.
func New(baseURL string, timeout time.Duration, retries int) *Client {
return &Client{
baseURL: baseURL,
http: &http.Client{Timeout: timeout},
retries: retries,
}
}
// Query asks the portal for outages matching the canonical address. The
// returned records are raw portal rows; callers must still filter by house.
func (c *Client) Query(ctx context.Context, canonical string) ([]outages.Outage, error) {
params := url.Values{}
params.Set("where", "address like '"+likePattern(canonical)+"'")
params.Set("outFields", "")
params.Set("returnGeometry", "false")
params.Set("f", "json")
var lastErr error
for attempt := 0; attempt <= c.retries; attempt++ {
if attempt > 0 {
backoff := time.Duration(attempt) * 150 * time.Millisecond
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(backoff):
}
}
rows, err := c.do(ctx, params)
if err == nil {
return rows, nil
}
lastErr = err
}
return nil, fmt.Errorf("portal query failed after %d attempts: %w", c.retries+1, lastErr)
}
func (c *Client) do(ctx context.Context, params url.Values) ([]outages.Outage, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"?"+params.Encode(), nil)
if err != nil {
return nil, fmt.Errorf("build request: %w", err)
}
resp, err := c.http.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("portal status %d", resp.StatusCode)
}
// Bound the response size to protect the service from a misbehaving portal.
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if err != nil {
return nil, err
}
return parseResponse(body)
}
// portalResponse mirrors the subset of the ArcGIS /query envelope we use.
type portalResponse struct {
Features []struct {
Value string `json:"value"`
Attributes map[string]any `json:"attributes"`
} `json:"features"`
Fields []struct {
Name string `json:"name"`
Domain struct {
CodedValues []struct {
Code any `json:"code"`
Name string `json:"name"`
} `json:"codedValues"`
} `json:"domain"`
} `json:"fields"`
}
func parseResponse(body []byte) ([]outages.Outage, error) {
var pr portalResponse
if err := json.Unmarshal(body, &pr); err != nil {
return nil, fmt.Errorf("decode portal response: %w", err)
}
// Build code -> name maps from the coded-value domains.
maps := map[string]map[string]string{}
for _, f := range pr.Fields {
if len(f.Domain.CodedValues) == 0 {
continue
}
m := make(map[string]string, len(f.Domain.CodedValues))
for _, cv := range f.Domain.CodedValues {
m[codeKey(cv.Code)] = cv.Name
}
maps[f.Name] = m
}
out := make([]outages.Outage, 0, len(pr.Features))
for _, feat := range pr.Features {
addr := cleanAddress(addressOf(feat.Attributes, feat.Value))
o := outages.Outage{
Address: addr,
Type: lookup(maps, "type_id", feat.Attributes),
System: lookup(maps, "system_id", feat.Attributes),
Time: lookup(maps, "time_category", feat.Attributes),
StartDate: stringOf(feat.Attributes["start_date"]),
EndDate: stringOf(feat.Attributes["end_date"]),
}
o.Action = actionFor(o.System)
// Parse the address at the boundary so the domain matches on structure
// and never depends on this portal's text format.
o.Unit = outages.HouseUnitOf(address.Parse(addr))
out = append(out, o)
}
return out, nil
}
func addressOf(attrs map[string]any, fallback string) string {
if v, ok := attrs["address"]; ok {
if s := stringOf(v); s != "" {
return s
}
}
return fallback
}
func lookup(maps map[string]map[string]string, field string, attrs map[string]any) string {
m, ok := maps[field]
if !ok {
return stringOf(attrs[field])
}
return m[codeKey(attrs[field])]
}
// codeKey normalises a coded value coming from JSON (float64) or a string.
func codeKey(v any) string {
switch t := v.(type) {
case nil:
return ""
case string:
return t
case float64:
return strconv.FormatInt(int64(t), 10)
case json.Number:
return t.String()
default:
return fmt.Sprintf("%v", v)
}
}
// stringOf renders an attribute value as a string. The important case is
// numeric dates: fmt's %v turns a float64 into scientific notation ("1.79e+12")
// which the date parser cannot read, so integers are formatted explicitly.
func stringOf(v any) string {
switch t := v.(type) {
case nil:
return ""
case string:
return t
case float64:
return strconv.FormatInt(int64(t), 10)
case json.Number:
return t.String()
default:
return fmt.Sprintf("%v", v)
}
}
// actionFor maps a portal system name to the canonical action code used by the
// scenario (kept for parity with the existing integration).
func actionFor(system string) string {
switch system {
case "Горячее водоснабжение":
return "waterWarm"
case "Холодное водоснабжение и канализация", "Холодное водоснабжение":
return "waterCold"
case "Электроснабжение":
return "electricitySupply"
case "Газоснабжение":
return "gasSupply"
case "Теплоснабжение":
return "heating"
default:
return ""
}
}
// likePattern builds the substring pattern the portal supports. Only pure
// numeric tokens (house numbers) are padded with spaces, so "1" does not
// incidentally match "10". Ordinals ("3-я") and houses with a letter ("24а")
// are left as-is: padding an ordinal breaks the match when it starts the
// string, and precision is guaranteed by the local house filter anyway.
func likePattern(canonical string) string {
parts := strings.Fields(canonical)
for i, part := range parts {
if isPureDigits(part) {
parts[i] = " " + part + " "
}
}
return "%" + strings.Join(parts, "%") + "%"
}
func isPureDigits(s string) bool {
if s == "" {
return false
}
for _, r := range s {
if r < '0' || r > '9' {
return false
}
}
return true
}
internal/adapters/arcgis/client_test.go
-128
View File
@@ -1,128 +0,0 @@
package arcgis
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
const envelope = `{
"features": [
{"value": "Станционная улица 1/1", "attributes": {
"address": "Станционная улица 1/1",
"type_id": 1, "system_id": 5, "time_category": 2,
"start_date": "2026-10-01T00:00:00", "end_date": "2026-10-31T00:00:00"
}}
],
"fields": [
{"name": "type_id", "domain": {"codedValues": [{"code": 1, "name": "Плановое"}]}},
{"name": "system_id", "domain": {"codedValues": [{"code": 5, "name": "Теплоснабжение"}]}},
{"name": "time_category", "domain": {"codedValues": [{"code": 2, "name": "Текущие"}]}}
]
}`
func TestQueryParsesEnvelope(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.URL.Query().Get("f"); got != "json" {
t.Errorf("missing f=json, got %q", got)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(envelope))
}))
defer ts.Close()
c := New(ts.URL, time.Second, 0)
rows, err := c.Query(context.Background(), "Станционная улица 1/1")
if err != nil {
t.Fatalf("Query: %v", err)
}
if len(rows) != 1 {
t.Fatalf("rows = %d, want 1", len(rows))
}
got := rows[0]
if got.Type != "Плановое" || got.System != "Теплоснабжение" || got.Time != "Текущие" {
t.Errorf("coded values not resolved: %+v", got)
}
if got.Action != "heating" {
t.Errorf("Action = %q, want heating", got.Action)
}
}
func TestQueryRetriesThenFails(t *testing.T) {
var calls int
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
http.Error(w, "boom", http.StatusInternalServerError)
}))
defer ts.Close()
c := New(ts.URL, time.Second, 2)
if _, err := c.Query(context.Background(), "Станционная улица 1/1"); err == nil {
t.Fatal("expected error")
}
if calls != 3 {
t.Errorf("calls = %d, want 3 (1 + 2 retries)", calls)
}
}
// TestQueryStripsDistrictSuffix is a regression test: the portal appends the
// district in parentheses, which used to hide the house number from the parser.
func TestQueryStripsDistrictSuffix(t *testing.T) {
const env = `{"features":[{"value":"Медкадры улица, 1 (Заельцовский район)","attributes":{"address":"Медкадры улица, 1 (Заельцовский район)"}}]}`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(env))
}))
defer ts.Close()
c := New(ts.URL, time.Second, 0)
rows, err := c.Query(context.Background(), "Медкадры улица 1")
if err != nil {
t.Fatalf("Query: %v", err)
}
if len(rows) != 1 {
t.Fatalf("rows = %d, want 1", len(rows))
}
if rows[0].Address != "Медкадры улица, 1" {
t.Errorf("district suffix not stripped: %q", rows[0].Address)
}
if rows[0].Unit.House != "1" {
t.Errorf("house not parsed at the boundary: %+v", rows[0].Unit)
}
}
// TestQueryFormatsNumericDates guards against scientific notation in dates,
// which broke the "current outage" filter.
func TestQueryFormatsNumericDates(t *testing.T) {
const env = `{"features":[{"value":"Ленина улица, 1","attributes":{"address":"Ленина улица, 1","start_date":1790820000000,"end_date":1791799200000}}]}`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(env))
}))
defer ts.Close()
rows, err := New(ts.URL, time.Second, 0).Query(context.Background(), "Ленина улица 1")
if err != nil {
t.Fatalf("Query: %v", err)
}
if rows[0].StartDate != "1790820000000" || rows[0].EndDate != "1791799200000" {
t.Errorf("numeric dates must be integer-formatted: %q..%q", rows[0].StartDate, rows[0].EndDate)
}
}
// TestLikePattern guards the ordinal/house padding rule: padding an ordinal
// that starts the string breaks the portal match.
func TestLikePattern(t *testing.T) {
got := likePattern("3-я Механическая улица 24а")
if strings.Contains(got, " 3-я ") {
t.Errorf("ordinal must not be space-padded: %q", got)
}
if strings.Contains(got, " 24а ") {
t.Errorf("house with a letter must not be space-padded: %q", got)
}
if !strings.Contains(likePattern("Ленина улица 1"), "% 1 %") {
t.Errorf("pure numeric house must be space-padded: %q", likePattern("Ленина улица 1"))
}
}
internal/adapters/httpapi/middleware.go
+25 -3
View File
@@ -14,10 +14,13 @@ type ctxKey int
const requestIDKey ctxKey = 0
// requestIDMW assigns a request id (for correlation in logs) and echoes it.
// A client-supplied id is accepted only when it is short and well-formed; the
// header ends up in logs, so accepting arbitrary bytes would be an injection
// and log-flooding vector.
func requestIDMW(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
id := r.Header.Get("X-Request-Id")
if id == "" {
if !validRequestID(id) {
id = newRequestID()
}
w.Header().Set("X-Request-Id", id)
@@ -26,6 +29,23 @@ func requestIDMW(next http.Handler) http.Handler {
})
}
const maxRequestIDLen = 128
func validRequestID(id string) bool {
if id == "" || len(id) > maxRequestIDLen {
return false
}
for _, r := range id {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
case r == '-', r == '_', r == '.':
default:
return false
}
}
return true
}
// loggingMW logs one line per request with method, path, status and duration.
// It records no addresses (PII) — only routing metadata.
func loggingMW(log *slog.Logger) func(http.Handler) http.Handler {
@@ -45,8 +65,10 @@ func loggingMW(log *slog.Logger) func(http.Handler) http.Handler {
}
}
// timeoutMW bounds the whole request, so a slow portal cannot tie up a
// goroutine indefinitely.
// timeoutMW bounds the whole request by setting a context deadline. Handlers
// and adapters that talk to the network honour it; purely CPU-bound handlers
// are additionally bounded by input limits and the HTTP server's read/write
// timeouts in cmd/addrkitd.
func timeoutMW(d time.Duration) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
internal/adapters/httpapi/server.go
+66 -66
View File
@@ -1,52 +1,54 @@
// Package httpapi exposes the addrkit domain over a small, versioned JSON API.
//
// It contains no business logic: handlers decode requests, call the resolver
// and the outages source, and encode responses. Cross-cutting concerns
// (timeouts, recovery, request ids, logging) live in middleware.
// and encode responses. Cross-cutting concerns (timeouts, recovery, request
// ids, logging) live in middleware.
package httpapi
import (
"context"
"encoding/json"
"errors"
"log/slog"
"net/http"
"time"
"unicode/utf8"
"log/slog"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
"git.totmin.ru/en2zmax/addrkit/internal/domain/outages"
)
// Resolver is the read side of the domain (implemented by *match.Resolver).
type Resolver interface {
Resolve(ctx context.Context, raw string) match.Result
ResolveInCity(ctx context.Context, raw, city string) match.Result
MatchCity(query string) (string, bool)
Version() string
Ready() bool
Cities() []string
}
// maxAddressRunes bounds user input. A real address is far shorter; the limit
// keeps a pathological string from turning into a CPU-bound fuzzy scan.
const maxAddressRunes = 512
// maxCityQueryRunes bounds the separate city-only endpoint.
const maxCityQueryRunes = 128
// Server wires the domain to HTTP.
type Server struct {
resolver Resolver
outages outages.Source
log *slog.Logger
now func() time.Time
loc *time.Location
timeout time.Duration
}
// New builds a server. loc controls how outage dates are interpreted; it
// should be the operational timezone (Asia/Novosibirsk).
func New(resolver Resolver, src outages.Source, log *slog.Logger, loc *time.Location) *Server {
if loc == nil {
loc = time.UTC
// New builds a server.
func New(resolver Resolver, log *slog.Logger) *Server {
if log == nil {
log = slog.Default()
}
return &Server{
resolver: resolver,
outages: src,
log: log,
now: time.Now,
loc: loc,
timeout: 10 * time.Second,
}
}
@@ -55,16 +57,21 @@ func New(resolver Resolver, src outages.Source, log *slog.Logger, loc *time.Loca
func (s *Server) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("POST /v1/resolve", s.handleResolve)
mux.HandleFunc("POST /v1/outages", s.handleOutages)
mux.HandleFunc("POST /v1/city", s.handleCity)
mux.HandleFunc("GET /v1/cities", s.handleCities)
mux.HandleFunc("GET /v1/healthz", s.handleHealthz)
mux.HandleFunc("GET /v1/readyz", s.handleReadyz)
mux.HandleFunc("GET /v1/version", s.handleVersion)
return recoverMW(s.log)(requestIDMW(loggingMW(s.log)(timeoutMW(s.timeout)(mux))))
// Logging sits outside recovery so a panic is still recorded as a 500.
return requestIDMW(loggingMW(s.log)(recoverMW(s.log)(timeoutMW(s.timeout)(mux))))
}
type resolveRequest struct {
Address string `json:"address"`
// City is an optional hint for clients that keep the city separate from
// the street line. It overrides a city found in the address text.
City string `json:"city"`
}
func (s *Server) handleResolve(w http.ResponseWriter, r *http.Request) {
@@ -72,64 +79,49 @@ func (s *Server) handleResolve(w http.ResponseWriter, r *http.Request) {
if !ok {
return
}
writeJSON(w, http.StatusOK, s.resolver.Resolve(r.Context(), req.Address))
writeJSON(w, http.StatusOK, s.resolver.ResolveInCity(r.Context(), req.Address, req.City))
}
type outagesResponse struct {
Status address.Status `json:"status"`
Address string `json:"address,omitempty"`
District string `json:"district,omitempty"`
Outages []outages.Outage `json:"outages"`
AsOf string `json:"as_of,omitempty"`
DataVersion string `json:"data_version,omitempty"`
Detail string `json:"detail,omitempty"`
type cityRequest struct {
Query string `json:"query"`
}
func (s *Server) handleOutages(w http.ResponseWriter, r *http.Request) {
req, ok := s.decode(w, r)
if !ok {
return
}
ctx := r.Context()
res := s.resolver.Resolve(ctx, req.Address)
type cityResponse struct {
Status string `json:"status"`
City string `json:"city,omitempty"`
}
out := outagesResponse{
Status: res.Status,
Address: res.Address,
District: res.District,
DataVersion: res.DataVersion,
Outages: []outages.Outage{},
}
if res.Status != address.StatusFound {
// No canonical address => nothing to query. Return the resolver's
// verdict so the caller can ask for clarification.
writeJSON(w, http.StatusOK, out)
// handleCity normalizes a city-only query ("нск", "г. Новосибирск",
// "Новосибирская область"). It is the small, separate use case for projects
// that need city names but not streets.
func (s *Server) handleCity(w http.ResponseWriter, r *http.Request) {
var req cityRequest
r.Body = http.MaxBytesReader(w, r.Body, 16*1024)
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
if s.outages == nil {
// The events module is optional and off when portal.url is empty.
out.Status = address.StatusInfra
out.Detail = "outages source is not configured"
writeJSON(w, http.StatusOK, out)
if req.Query == "" {
writeError(w, http.StatusBadRequest, "query is required")
return
}
if utf8.RuneCountInString(req.Query) > maxCityQueryRunes {
writeError(w, http.StatusBadRequest, "query is too long")
return
}
if city, ok := s.resolver.MatchCity(req.Query); ok {
writeJSON(w, http.StatusOK, cityResponse{Status: "found", City: city})
return
}
writeJSON(w, http.StatusOK, cityResponse{Status: "not_found"})
}
raw, err := s.outages.Query(ctx, res.Address)
if err != nil {
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
out.Status = address.StatusPortalTimeout
} else {
out.Status = address.StatusInfra
}
writeJSON(w, http.StatusOK, out)
return
func (s *Server) handleCities(w http.ResponseWriter, _ *http.Request) {
cities := s.resolver.Cities()
if cities == nil {
cities = []string{}
}
parsed := address.Parse(res.Address)
current := outages.Current(raw, s.now(), s.loc)
out.Outages = outages.MatchingHouse(current, parsed)
out.AsOf = s.now().In(s.loc).Format(time.RFC3339)
writeJSON(w, http.StatusOK, out)
writeJSON(w, http.StatusOK, map[string][]string{"cities": cities})
}
func (s *Server) handleHealthz(w http.ResponseWriter, _ *http.Request) {
@@ -160,6 +152,14 @@ func (s *Server) decode(w http.ResponseWriter, r *http.Request) (resolveRequest,
writeError(w, http.StatusBadRequest, "address is required")
return resolveRequest{}, false
}
if utf8.RuneCountInString(req.Address) > maxAddressRunes {
writeError(w, http.StatusBadRequest, "address is too long")
return resolveRequest{}, false
}
if utf8.RuneCountInString(req.City) > 128 {
writeError(w, http.StatusBadRequest, "city is too long")
return resolveRequest{}, false
}
return req, true
}
internal/adapters/httpapi/server_test.go
+137 -48
View File
@@ -2,46 +2,51 @@ package httpapi
import (
"bytes"
"context"
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
"git.totmin.ru/en2zmax/addrkit/internal/domain/outages"
)
func entry(t *testing.T, city, raw string) address.Entry {
t.Helper()
p := address.NewParser(address.NewDictionary(cityDictionary()), "").ParseInCity(raw, city)
return address.Entry{
ID: raw, Address: raw, District: "Тестовый",
City: p.City, CityKey: p.CityKey,
Ordinal: p.Ordinal, NameKey: p.NameKey, NameDisplay: p.NameDisplay,
StreetType: p.StreetType, StreetKey: p.StreetKey,
House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter,
Building: p.Building, BuildingFrac: p.BuildingFrac, BuildingLetter: p.BuildingLetter,
Structure: p.Structure,
}
}
func cityDictionary() []address.City {
return []address.City{
{Name: "Новосибирск", Aliases: []string{"нск", "новосиб", "новосибирская область"}},
{Name: "Бердск"},
}
}
func testResolver(t *testing.T) *match.Resolver {
t.Helper()
var entries []address.Entry
for _, a := range []string{"Станционная улица 1/1", "2-я Станционная улица 1/1", "Искитимская улица 5"} {
p := address.Parse(a)
entries = append(entries, address.Entry{
ID: a, Address: a, District: "Тестовый",
Ordinal: p.Ordinal, NameKey: p.NameKey, StreetType: p.StreetType, StreetKey: p.StreetKey,
House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter,
})
entries = append(entries, entry(t, "", a))
}
return match.NewResolver(match.NewIndex("test-version", entries, match.DefaultOptions()), nil)
}
type fakeSource struct {
list []outages.Outage
err error
}
func (f fakeSource) Query(context.Context, string) ([]outages.Outage, error) {
return f.list, f.err
}
func newServer(t *testing.T, src fakeSource) *httptest.Server {
func newServer(t *testing.T, resolver Resolver) *httptest.Server {
t.Helper()
srv := New(testResolver(t), src, slog.New(slog.NewTextHandler(io.Discard, nil)), time.UTC)
srv := New(resolver, slog.New(slog.NewTextHandler(io.Discard, nil)))
ts := httptest.NewServer(srv.Handler())
t.Cleanup(ts.Close)
return ts
@@ -57,7 +62,7 @@ func postJSON(t *testing.T, url, body string) *http.Response {
}
func TestResolveEndpoint(t *testing.T) {
ts := newServer(t, fakeSource{})
ts := newServer(t, testResolver(t))
resp := postJSON(t, ts.URL+"/v1/resolve", `{"address":"2-я Станционная 1/1"}`)
defer resp.Body.Close()
@@ -73,38 +78,120 @@ func TestResolveEndpoint(t *testing.T) {
}
}
func TestOutagesEndpointFiltersHouse(t *testing.T) {
src := fakeSource{list: []outages.Outage{
{Address: "Станционная улица 1/1", System: "Теплоснабжение"},
{Address: "Станционная улица 1/2", System: "Теплоснабжение"},
}}
ts := newServer(t, src)
resp := postJSON(t, ts.URL+"/v1/outages", `{"address":"станционная 1/1"}`)
defer resp.Body.Close()
var out outagesResponse
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
t.Fatalf("decode: %v", err)
}
if out.Status != address.StatusFound {
t.Fatalf("status = %q, want found", out.Status)
}
if len(out.Outages) != 1 || out.Outages[0].Address != "Станционная улица 1/1" {
t.Fatalf("house filter failed: %+v", out.Outages)
func TestResolveValidation(t *testing.T) {
ts := newServer(t, testResolver(t))
for _, body := range []string{`{}`, `{"address":""}`, `not json`} {
resp := postJSON(t, ts.URL+"/v1/resolve", body)
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("body %q: status = %d, want 400", body, resp.StatusCode)
}
}
}
func TestResolveValidation(t *testing.T) {
ts := newServer(t, fakeSource{})
resp := postJSON(t, ts.URL+"/v1/resolve", `{}`)
defer resp.Body.Close()
func TestResolveRejectsOversizedAddress(t *testing.T) {
ts := newServer(t, testResolver(t))
body, _ := json.Marshal(map[string]string{"address": strings.Repeat("а", maxAddressRunes+1)})
resp := postJSON(t, ts.URL+"/v1/resolve", string(body))
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("status = %d, want 400", resp.StatusCode)
}
}
func TestResolveCityHint(t *testing.T) {
entries := []address.Entry{
entry(t, "Новосибирск", "Ленина улица 5"),
entry(t, "Бердск", "Ленина улица 5"),
}
resolver := match.NewResolver(
match.NewIndex("test", entries, match.DefaultOptions()),
nil,
)
ts := newServer(t, resolver)
resp := postJSON(t, ts.URL+"/v1/resolve", `{"address":"Ленина 5","city":"Бердск"}`)
defer resp.Body.Close()
var res match.Result
if err := json.NewDecoder(resp.Body).Decode(&res); err != nil {
t.Fatalf("decode: %v", err)
}
if res.Status != address.StatusFound || res.City != "Бердск" || res.Address != "Ленина улица 5" {
t.Fatalf("unexpected result: %+v", res)
}
}
func TestCitiesEndpoint(t *testing.T) {
entries := []address.Entry{
entry(t, "Новосибирск", "Ленина улица 5"),
entry(t, "Бердск", "Ленина улица 5"),
}
resolver := match.NewResolver(match.NewIndex("test", entries, match.DefaultOptions()), nil)
ts := newServer(t, resolver)
resp, err := http.Get(ts.URL + "/v1/cities")
if err != nil {
t.Fatalf("get: %v", err)
}
defer resp.Body.Close()
var body struct {
Cities []string `json:"cities"`
}
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
t.Fatalf("decode: %v", err)
}
if len(body.Cities) != 2 || body.Cities[0] != "Бердск" || body.Cities[1] != "Новосибирск" {
t.Fatalf("cities = %v", body.Cities)
}
}
func TestCityEndpoint(t *testing.T) {
parser := address.NewParser(address.NewDictionary(cityDictionary()), "")
resolver := match.NewResolverWithOptions(
match.NewIndex("test", nil, match.DefaultOptions()),
nil,
match.ResolverOptions{Parser: parser},
)
ts := newServer(t, resolver)
for _, tt := range []struct {
query string
city string
}{
{"нск", "Новосибирск"},
{"г. Новосибирск", "Новосибирск"},
{"Новосибирская область", "Новосибирск"},
{"Бердск", "Бердск"},
{"Томск", ""},
} {
body, _ := json.Marshal(map[string]string{"query": tt.query})
resp := postJSON(t, ts.URL+"/v1/city", string(body))
var got cityResponse
if err := json.NewDecoder(resp.Body).Decode(&got); err != nil {
resp.Body.Close()
t.Fatalf("%q: decode: %v", tt.query, err)
}
resp.Body.Close()
if tt.city == "" {
if got.Status != "not_found" {
t.Errorf("%q: status = %q, want not_found", tt.query, got.Status)
}
continue
}
if got.Status != "found" || got.City != tt.city {
t.Errorf("%q: %+v", tt.query, got)
}
}
resp := postJSON(t, ts.URL+"/v1/city", `{}`)
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("empty query status = %d, want 400", resp.StatusCode)
}
}
func TestReadyzAndVersion(t *testing.T) {
ts := newServer(t, fakeSource{})
ts := newServer(t, testResolver(t))
resp, err := http.Get(ts.URL + "/v1/readyz")
if err != nil {
@@ -120,9 +207,11 @@ func TestReadyzAndVersion(t *testing.T) {
t.Fatalf("version: %v", err)
}
defer resp.Body.Close()
var v map[string]string
_ = json.NewDecoder(resp.Body).Decode(&v)
if v["data_version"] != "test-version" {
t.Fatalf("version = %q", v["data_version"])
var body map[string]string
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
t.Fatalf("decode: %v", err)
}
if body["data_version"] != "test-version" {
t.Fatalf("data_version = %q", body["data_version"])
}
}
internal/adapters/llm/rewriter.go
+14 -3
View File
@@ -14,10 +14,12 @@ import (
_ "embed"
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"time"
"unicode/utf8"
"git.totmin.ru/en2zmax/addrkit/internal/config"
)
@@ -128,7 +130,9 @@ func (r *Rewriter) Rewrite(ctx context.Context, raw string) ([]string, error) {
}
var cr chatResponse
if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil {
// Bound the reply: a broken or hostile model must not be able to stream an
// unbounded JSON body into the service.
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&cr); err != nil {
return nil, fmt.Errorf("decode llm response: %w", err)
}
if len(cr.Choices) == 0 {
@@ -154,8 +158,15 @@ func parseVariants(content string) []string {
}
out := make([]string, 0, len(parsed.Variants))
for _, v := range parsed.Variants {
if s := strings.TrimSpace(v); s != "" {
out = append(out, s)
s := strings.TrimSpace(v)
// A variant is a short address; the cap also keeps a hostile model
// from making the resolver scan a megabyte-long token.
if s == "" || utf8.RuneCountInString(s) > 512 {
continue
}
out = append(out, s)
if len(out) == 3 {
break
}
}
return out
internal/adapters/llm/rewriter_test.go
+14
View File
@@ -86,3 +86,17 @@ func TestParseVariants(t *testing.T) {
})
}
}
func TestParseVariantsCapsCountAndLength(t *testing.T) {
long := strings.Repeat("x", 600)
content := `{"variants":["` + long + `","a","b","c","d"]}`
out := parseVariants(content)
if len(out) != 3 {
t.Fatalf("variants = %v, want 3", out)
}
for _, v := range out {
if len([]rune(v)) > 512 {
t.Fatalf("oversized variant accepted: %d runes", len([]rune(v)))
}
}
}
internal/adapters/registrystore/sqlite.go
+112 -43
View File
@@ -5,37 +5,49 @@ package registrystore
import (
"database/sql"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
_ "modernc.org/sqlite" // registers the "sqlite" driver
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
)
// SchemaVersion identifies the on-disk format. It is independent of the data
// version (the registry snapshot date) and must be bumped whenever the key
// format or columns change. A service refuses to open a database it does not
// understand instead of silently resolving against half-empty fields.
const SchemaVersion = "2"
const schema = `
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS address (
id TEXT PRIMARY KEY,
address TEXT NOT NULL,
district TEXT NOT NULL DEFAULT '',
ordinal TEXT NOT NULL DEFAULT '',
name_key TEXT NOT NULL DEFAULT '',
name_display TEXT NOT NULL DEFAULT '',
street_type TEXT NOT NULL DEFAULT '',
street_key TEXT NOT NULL,
house TEXT NOT NULL DEFAULT '',
house_frac TEXT NOT NULL DEFAULT '',
house_letter TEXT NOT NULL DEFAULT '',
building TEXT NOT NULL DEFAULT '',
building_frac TEXT NOT NULL DEFAULT '',
id TEXT PRIMARY KEY,
address TEXT NOT NULL,
district TEXT NOT NULL DEFAULT '',
city TEXT NOT NULL DEFAULT '',
city_key TEXT NOT NULL DEFAULT '',
ordinal TEXT NOT NULL DEFAULT '',
name_key TEXT NOT NULL DEFAULT '',
name_display TEXT NOT NULL DEFAULT '',
street_type TEXT NOT NULL DEFAULT '',
street_key TEXT NOT NULL,
house TEXT NOT NULL DEFAULT '',
house_frac TEXT NOT NULL DEFAULT '',
house_letter TEXT NOT NULL DEFAULT '',
building TEXT NOT NULL DEFAULT '',
building_frac TEXT NOT NULL DEFAULT '',
building_letter TEXT NOT NULL DEFAULT '',
structure TEXT NOT NULL DEFAULT ''
structure TEXT NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_address_street ON address(street_key);
CREATE INDEX IF NOT EXISTS idx_address_city ON address(city_key);
`
// Save writes entries to path atomically: it builds a temporary file and
@@ -44,13 +56,21 @@ CREATE INDEX IF NOT EXISTS idx_address_street ON address(street_key);
// The build uses the rollback journal (not WAL) on purpose, so that the result
// is a single file with no companion -wal/-shm that a rename would orphan.
func Save(path, version string, entries []address.Entry) error {
tmp := path + ".tmp"
removeDBFiles(tmp)
if len(entries) == 0 {
return errors.New("refusing to publish an empty index")
}
db, err := sql.Open("sqlite", "file:"+tmp+
"?_pragma=journal_mode(DELETE)"+
"&_pragma=synchronous(FULL)"+
"&_pragma=busy_timeout(5000)")
tmp := path + ".tmp"
// A previous failed build may have left a complete temp database with the
// same meta rows; reusing it would hit a primary-key conflict.
_ = os.Remove(tmp)
removeSidecars(tmp)
uri, err := dsn(tmp, false)
if err != nil {
return err
}
db, err := sql.Open("sqlite", uri)
if err != nil {
return fmt.Errorf("open temp db: %w", err)
}
@@ -68,12 +88,46 @@ func Save(path, version string, entries []address.Entry) error {
if err := os.Rename(tmp, path); err != nil {
return fmt.Errorf("publish db: %w", err)
}
// Drop only the temp file's journal siblings; the published base file must
// survive.
// The rename is atomic, but a crash immediately after it may lose the
// directory entry. Syncing the parent makes the publish durable.
syncDir(filepath.Dir(path))
removeSidecars(tmp)
return nil
}
// dsn builds a SQLite URI. Concatenating "file:" + path breaks on paths that
// contain "?", "#" or "%", so let net/url escape the path for us.
func dsn(path string, readOnly bool) (string, error) {
abs, err := filepath.Abs(path)
if err != nil {
return "", fmt.Errorf("resolve db path: %w", err)
}
u := url.URL{Scheme: "file", Path: abs}
q := u.Query()
q.Add("_pragma", "busy_timeout(5000)")
if readOnly {
q.Set("mode", "ro")
q.Add("_pragma", "query_only(1)")
} else {
q.Add("_pragma", "journal_mode(DELETE)")
q.Add("_pragma", "synchronous(FULL)")
}
u.RawQuery = q.Encode()
return u.String(), nil
}
// syncDir fsyncs a directory so a freshly renamed file survives a crash. It is
// best effort: some filesystems do not support directory fsync, and failing to
// sync here is no worse than not trying.
func syncDir(dir string) {
f, err := os.Open(dir)
if err != nil {
return
}
defer f.Close()
_ = f.Sync()
}
func insertAll(db *sql.DB, version string, entries []address.Entry) error {
tx, err := db.Begin()
if err != nil {
@@ -84,11 +138,15 @@ func insertAll(db *sql.DB, version string, entries []address.Entry) error {
if _, err := tx.Exec(`INSERT INTO meta(key, value) VALUES ('version', ?)`, version); err != nil {
return fmt.Errorf("insert meta: %w", err)
}
if _, err := tx.Exec(`INSERT INTO meta(key, value) VALUES ('schema_version', ?)`, SchemaVersion); err != nil {
return fmt.Errorf("insert schema version: %w", err)
}
stmt, err := tx.Prepare(`INSERT INTO address (
id, address, district, ordinal, name_key, name_display, street_type, street_key,
house, house_frac, house_letter, building, building_frac, building_letter, structure
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
id, address, district, city, city_key, ordinal, name_key, name_display,
street_type, street_key, house, house_frac, house_letter, building,
building_frac, building_letter, structure
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
if err != nil {
return fmt.Errorf("prepare insert: %w", err)
}
@@ -101,8 +159,9 @@ func insertAll(db *sql.DB, version string, entries []address.Entry) error {
id = fmt.Sprintf("row-%d", i)
}
if _, err := stmt.Exec(
id, e.Address, e.District, e.Ordinal, e.NameKey, e.NameDisplay, e.StreetType, e.StreetKey,
e.House, e.HouseFrac, e.HouseLetter, e.Building, e.BuildingFrac, e.BuildingLetter, e.Structure,
id, e.Address, e.District, e.City, e.CityKey, e.Ordinal, e.NameKey, e.NameDisplay,
e.StreetType, e.StreetKey, e.House, e.HouseFrac, e.HouseLetter, e.Building,
e.BuildingFrac, e.BuildingLetter, e.Structure,
); err != nil {
return fmt.Errorf("insert %q: %w", e.Address, err)
}
@@ -114,26 +173,41 @@ func insertAll(db *sql.DB, version string, entries []address.Entry) error {
}
// Load opens path read-only and returns the data version and all entries.
// It fails closed when the file was produced by an incompatible schema.
func Load(path string) (version string, entries []address.Entry, err error) {
if _, err := os.Stat(path); err != nil {
return "", nil, fmt.Errorf("stat db: %w", err)
}
db, err := sql.Open("sqlite", "file:"+path+
"?mode=ro"+
"&_pragma=query_only(1)"+
"&_pragma=busy_timeout(5000)")
uri, err := dsn(path, true)
if err != nil {
return "", nil, err
}
db, err := sql.Open("sqlite", uri)
if err != nil {
return "", nil, fmt.Errorf("open db: %w", err)
}
defer db.Close()
var schemaVersion string
if err := db.QueryRow(`SELECT value FROM meta WHERE key='schema_version'`).Scan(&schemaVersion); err != nil {
return "", nil, fmt.Errorf("read schema version (re-run addrkit seed): %w", err)
}
if schemaVersion != SchemaVersion {
return "", nil, fmt.Errorf(
"unsupported index schema %q (want %q); re-run addrkit seed",
schemaVersion, SchemaVersion,
)
}
if err := db.QueryRow(`SELECT value FROM meta WHERE key='version'`).Scan(&version); err != nil {
return "", nil, fmt.Errorf("read version: %w", err)
}
rows, err := db.Query(`SELECT id, address, district, ordinal, name_key, name_display,
street_type, street_key, house, house_frac, house_letter, building, building_frac,
building_letter, structure FROM address`)
rows, err := db.Query(`SELECT id, address, district, city, city_key, ordinal,
name_key, name_display, street_type, street_key, house, house_frac,
house_letter, building, building_frac, building_letter, structure
FROM address
ORDER BY city_key, street_key, house, house_frac, house_letter,
building, building_frac, building_letter, structure, address, id`)
if err != nil {
return "", nil, fmt.Errorf("query: %w", err)
}
@@ -142,9 +216,10 @@ func Load(path string) (version string, entries []address.Entry, err error) {
for rows.Next() {
var e address.Entry
if err := rows.Scan(
&e.ID, &e.Address, &e.District, &e.Ordinal, &e.NameKey, &e.NameDisplay,
&e.StreetType, &e.StreetKey, &e.House, &e.HouseFrac, &e.HouseLetter,
&e.Building, &e.BuildingFrac, &e.BuildingLetter, &e.Structure,
&e.ID, &e.Address, &e.District, &e.City, &e.CityKey, &e.Ordinal,
&e.NameKey, &e.NameDisplay, &e.StreetType, &e.StreetKey, &e.House,
&e.HouseFrac, &e.HouseLetter, &e.Building, &e.BuildingFrac,
&e.BuildingLetter, &e.Structure,
); err != nil {
return "", nil, fmt.Errorf("scan: %w", err)
}
@@ -156,12 +231,6 @@ func Load(path string) (version string, entries []address.Entry, err error) {
return version, entries, nil
}
// removeDBFiles deletes a database file and its journal siblings.
func removeDBFiles(base string) {
_ = os.Remove(base)
removeSidecars(base)
}
// removeSidecars deletes only the -wal/-shm companions of a database file.
func removeSidecars(base string) {
_ = os.Remove(base + "-wal")
internal/adapters/registrystore/sqlite_test.go
+58 -2
View File
@@ -1,7 +1,9 @@
package registrystore
import (
"database/sql"
"path/filepath"
"strings"
"testing"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
@@ -11,11 +13,13 @@ func TestSaveLoadRoundTrip(t *testing.T) {
path := filepath.Join(t.TempDir(), "registry.db")
addr := "Станционная улица 1/1"
p := address.Parse(addr)
parser := address.NewParser(address.NewDictionary([]address.City{{Name: "Новосибирск"}}), "")
p := parser.ParseInCity(addr, "Новосибирск")
entries := []address.Entry{{
ID: "a1", Address: addr, District: "Ленинский",
City: p.City, CityKey: p.CityKey,
Ordinal: p.Ordinal, NameKey: p.NameKey, StreetType: p.StreetType, StreetKey: p.StreetKey,
House: p.House, HouseFrac: p.HouseFrac,
House: p.House, HouseFrac: p.HouseFrac, Structure: "2",
}}
if err := Save(path, "2026-10-09", entries); err != nil {
@@ -35,6 +39,58 @@ func TestSaveLoadRoundTrip(t *testing.T) {
if got[0].Address != addr || got[0].District != "Ленинский" || got[0].StreetKey != p.StreetKey {
t.Errorf("round-tripped entry mismatch: %+v", got[0])
}
if got[0].City != "Новосибирск" || got[0].CityKey != "новосибирск" || got[0].Structure != "2" {
t.Errorf("city/structure not round-tripped: %+v", got[0])
}
}
func TestSaveRejectsEmptyIndex(t *testing.T) {
path := filepath.Join(t.TempDir(), "registry.db")
if err := Save(path, "v1", nil); err == nil {
t.Fatal("expected error for empty index")
}
}
func TestLoadRejectsIncompatibleSchema(t *testing.T) {
path := filepath.Join(t.TempDir(), "old.db")
uri, err := dsn(path, false)
if err != nil {
t.Fatalf("dsn: %v", err)
}
db, err := sql.Open("sqlite", uri)
if err != nil {
t.Fatalf("open: %v", err)
}
if _, err := db.Exec(`CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`); err != nil {
t.Fatalf("create meta: %v", err)
}
if _, err := db.Exec(`INSERT INTO meta(key, value) VALUES ('schema_version', '1')`); err != nil {
t.Fatalf("insert schema version: %v", err)
}
db.Close()
_, _, err = Load(path)
if err == nil || !strings.Contains(err.Error(), "unsupported index schema") {
t.Fatalf("Load error = %v, want unsupported schema", err)
}
}
// TestPathWithSpecialCharacters guards the SQLite URI construction: a naive
// "file:" + path breaks on ?, # and % in the path.
func TestPathWithSpecialCharacters(t *testing.T) {
path := filepath.Join(t.TempDir(), "registry ? # 100%.db")
entries := []address.Entry{{ID: "x", Address: "Первая улица 1"}}
if err := Save(path, "v1", entries); err != nil {
t.Fatalf("Save: %v", err)
}
version, got, err := Load(path)
if err != nil {
t.Fatalf("Load: %v", err)
}
if version != "v1" || len(got) != 1 {
t.Fatalf("version=%q entries=%+v", version, got)
}
}
// TestSaveIsAtomicOverwrite verifies that re-seeding replaces the previous data