diff --git a/README.md b/README.md
index 596d833..974ef1b 100644
--- a/README.md
+++ b/README.md
@@ -87,6 +87,11 @@ CLI без сервиса:
незаданная переменная = ошибка запуска (fail closed). Секреты — только из
окружения.
+Промпт LLM-rewriter **не зашит в код**: встроенный дефолт лежит в
+`internal/adapters/llm/default_prompt.txt` и компилируется в бинарь (`go:embed`).
+Переопределить можно в конфиге: `llm.system_prompt` (инлайн) или
+`llm.system_prompt_file` (путь к файлу; приоритетнее инлайна).
+
## Принятые решения
- **Детерминизм.** Один вход — один выход, порядок кандидатов стабилен.
diff --git a/cmd/addrkit/main.go b/cmd/addrkit/main.go
index b0c76bc..5de2198 100644
--- a/cmd/addrkit/main.go
+++ b/cmd/addrkit/main.go
@@ -10,8 +10,12 @@ import (
"fmt"
"math/rand"
"os"
+ "strings"
+ "time"
+ "git.totmin.ru/en2zmax/addrkit/internal/adapters/llm"
"git.totmin.ru/en2zmax/addrkit/internal/adapters/registrystore"
+ "git.totmin.ru/en2zmax/addrkit/internal/config"
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
"git.totmin.ru/en2zmax/addrkit/internal/eval"
"git.totmin.ru/en2zmax/addrkit/internal/seed"
@@ -131,6 +135,9 @@ func runVerify(args []string) int {
func runResolve(args []string) int {
fs := flag.NewFlagSet("resolve", flag.ExitOnError)
db := fs.String("db", "", "SQLite database")
+ llmURL := fs.String("llm-url", "", "enable the LLM rewriter with this base URL")
+ llmModel := fs.String("llm-model", "qwen3-14b-awq", "LLM model name")
+ llmToken := fs.String("llm-token", "", "LLM auth token")
_ = fs.Parse(args)
rest := fs.Args()
if *db == "" || len(rest) == 0 {
@@ -142,7 +149,14 @@ func runResolve(args []string) int {
fmt.Fprintf(os.Stderr, "resolve: %v\n", err)
return 1
}
- resolver := match.NewResolver(match.NewIndex(version, entries, match.DefaultOptions()), nil)
+ var rewriter match.Rewriter
+ if *llmURL != "" {
+ rewriter = llm.New(config.LLMConfig{
+ Enabled: true, URL: *llmURL, Model: *llmModel, Token: *llmToken,
+ TimeoutSeconds: 5, MaxTokens: 512, DisableThinking: true,
+ })
+ }
+ resolver := match.NewResolver(match.NewIndex(version, entries, match.DefaultOptions()), rewriter)
res := resolver.Resolve(context.Background(), rest[0])
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
@@ -152,11 +166,17 @@ func runResolve(args []string) int {
// runEval generates citizen-style perturbations from the seeded registry and
// reports how well the resolver copes. Results are deterministic per -seed.
+// When -llm-url is set, the optional LLM rewriter is enabled and its impact is
+// measured on the same cases.
func runEval(args []string) int {
fs := flag.NewFlagSet("eval", flag.ExitOnError)
db := fs.String("db", "", "SQLite database")
sample := fs.Int("sample", 500, "number of registry entries to sample")
seedVal := fs.Int64("seed", 1, "random seed")
+ llmURL := fs.String("llm-url", "", "enable the LLM rewriter with this base URL")
+ llmModel := fs.String("llm-model", "qwen3-14b-awq", "LLM model name")
+ llmToken := fs.String("llm-token", "", "LLM auth token")
+ categories := fs.String("categories", "", "comma-separated category filter (default: all)")
_ = fs.Parse(args)
if *db == "" {
fmt.Fprintln(os.Stderr, "eval: -db is required")
@@ -168,12 +188,80 @@ func runEval(args []string) int {
return 1
}
ix := match.NewIndex("eval", entries, match.DefaultOptions())
+
+ var rewriter match.Rewriter
+ var counter *countingRewriter
+ if *llmURL != "" {
+ counter = &countingRewriter{inner: llm.New(config.LLMConfig{
+ Enabled: true,
+ URL: *llmURL,
+ Model: *llmModel,
+ Token: *llmToken,
+ TimeoutSeconds: 5,
+ MaxTokens: 512,
+ DisableThinking: true,
+ })}
+ rewriter = counter
+ fmt.Fprintf(os.Stderr, "llm rewriter enabled: %s (%s)\n", *llmURL, *llmModel)
+ }
+ resolver := match.NewResolver(ix, rewriter)
+
rng := rand.New(rand.NewSource(*seedVal))
cases := eval.Generate(entries, *sample, rng)
- report := eval.Run(ix, cases)
+ cases = filterCategories(cases, *categories)
+ fmt.Fprintf(os.Stderr, "cases: %d\n", len(cases))
+
+ report := eval.Run(resolver, cases)
+
+ if counter != nil {
+ fmt.Fprintf(os.Stderr, "llm calls: %d, avg latency: %s\n", counter.calls, counter.avg())
+ }
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
_ = enc.Encode(report)
return 0
}
+
+// filterCategories keeps only the requested categories (all when spec is empty).
+func filterCategories(cases []eval.Case, spec string) []eval.Case {
+ if strings.TrimSpace(spec) == "" {
+ return cases
+ }
+ want := map[string]bool{}
+ for _, c := range strings.Split(spec, ",") {
+ if c = strings.TrimSpace(c); c != "" {
+ want[c] = true
+ }
+ }
+ out := cases[:0:0]
+ for _, c := range cases {
+ if want[c.Category] {
+ out = append(out, c)
+ }
+ }
+ return out
+}
+
+// countingRewriter records how often and how long the rewriter is consulted,
+// so the cost of the LLM path is visible in benchmarks.
+type countingRewriter struct {
+ inner match.Rewriter
+ calls int
+ total time.Duration
+}
+
+func (c *countingRewriter) Rewrite(ctx context.Context, raw string) ([]string, error) {
+ start := time.Now()
+ out, err := c.inner.Rewrite(ctx, raw)
+ c.calls++
+ c.total += time.Since(start)
+ return out, err
+}
+
+func (c *countingRewriter) avg() time.Duration {
+ if c.calls == 0 {
+ return 0
+ }
+ return c.total / time.Duration(c.calls)
+}
diff --git a/config.example.json b/config.example.json
index 51a31b5..1e988b9 100644
--- a/config.example.json
+++ b/config.example.json
@@ -17,6 +17,10 @@
"url": "${ADDRKIT_LLM_URL}",
"token": "${ADDRKIT_LLM_TOKEN}",
"model": "qwen3-14b-awq",
- "timeout_seconds": 3
+ "timeout_seconds": 3,
+ "max_tokens": 512,
+ "disable_thinking": true,
+ "system_prompt": "",
+ "system_prompt_file": ""
}
}
diff --git a/dev/config.llm.json b/dev/config.llm.json
new file mode 100644
index 0000000..bc8da6c
--- /dev/null
+++ b/dev/config.llm.json
@@ -0,0 +1,8 @@
+{
+ "addr": "127.0.0.1:6172",
+ "data_path": "dev/registry.db",
+ "log_level": "warn",
+ "resolve": { "fuzzy_threshold": 0.82, "ambiguity_margin": 0.06, "max_candidates": 5 },
+ "portal": { "url": "https://map.novo-sibirsk.ru/elitegis/rest/services/maps/disconnections/MapServer/101/query", "timeout_seconds": 2, "retries": 0 },
+ "llm": { "enabled": true, "url": "http://127.0.0.1:6193", "token": "EMPTY", "model": "qwen3-14b-awq", "timeout_seconds": 5, "max_tokens": 512, "disable_thinking": true }
+}
diff --git a/internal/adapters/httpapi/server_test.go b/internal/adapters/httpapi/server_test.go
index e05bfc7..6eafb66 100644
--- a/internal/adapters/httpapi/server_test.go
+++ b/internal/adapters/httpapi/server_test.go
@@ -19,7 +19,7 @@ import (
func testResolver(t *testing.T) *match.Resolver {
t.Helper()
var entries []address.Entry
- for _, a := range []string{"Станционная улица 1/1", "Искитимская улица 5"} {
+ for _, a := range []string{"Станционная улица 1/1", "2-я Станционная улица 1/1", "Искитимская улица 5"} {
p := address.Parse(a)
entries = append(entries, address.Entry{
ID: a, Address: a, District: "Тестовый",
@@ -68,7 +68,7 @@ func TestResolveEndpoint(t *testing.T) {
if err := json.NewDecoder(resp.Body).Decode(&res); err != nil {
t.Fatalf("decode: %v", err)
}
- if res.Status != address.StatusFound || res.Address != "Станционная улица 1/1" {
+ if res.Status != address.StatusFound || res.Address != "2-я Станционная улица 1/1" {
t.Fatalf("unexpected result: %+v", res)
}
}
diff --git a/internal/adapters/llm/default_prompt.txt b/internal/adapters/llm/default_prompt.txt
new file mode 100644
index 0000000..96a7820
--- /dev/null
+++ b/internal/adapters/llm/default_prompt.txt
@@ -0,0 +1,11 @@
+Ты — нормализатор российских адресов. На входе адрес, который человек написал как угодно: с опечатками, перестановками букв, сокращениями и лишними словами.
+Верни СТРОГО JSON: {"variants":["...","..."]} — до 3 вариантов ИСПРАВЛЕННОЙ записи того же адреса, от самого вероятного к менее вероятному.
+Формат варианта: "<Название улицы> <дом>[/<дробь>][ <литера>][ к<корпус>][ стр.<строение>]".
+Правила:
+- исправляй опечатки и перестановки букв в названии улицы;
+- раскрывай сокращения: ул.→улица, пер.→переулок, пр-т→проспект, ш.→шоссе, д.→дом;
+- восстанавливай порядковые: "2 я", "вторая" → "2-я";
+- убирай "город", "район", "квартира" и прочий мусор;
+- НЕ заменяй улицу на другую и НЕ добавляй дом, которого нет во входе: меняй только написание;
+- если адрес уже верный — верни его без изменений.
+Отвечай только JSON, без пояснений.
diff --git a/internal/adapters/llm/rewriter.go b/internal/adapters/llm/rewriter.go
index 0b50958..3434987 100644
--- a/internal/adapters/llm/rewriter.go
+++ b/internal/adapters/llm/rewriter.go
@@ -11,21 +11,33 @@ package llm
import (
"bytes"
"context"
+ _ "embed"
"encoding/json"
"fmt"
"net/http"
+ "regexp"
"strings"
"time"
"git.totmin.ru/en2zmax/addrkit/internal/config"
)
+// defaultSystemPrompt is embedded from a readable text file so the prompt can
+// be reviewed and versioned like documentation. It can be overridden per
+// deployment via llm.system_prompt or llm.system_prompt_file in the config.
+//
+//go:embed default_prompt.txt
+var defaultSystemPrompt string
+
// Rewriter asks a model for normalised variants of an address.
type Rewriter struct {
- endpoint string
- token string
- model string
- http *http.Client
+ endpoint string
+ token string
+ model string
+ systemPrompt string
+ maxTokens int
+ disableThinking bool
+ http *http.Client
}
// New builds a rewriter from configuration. The URL is treated as a base; the
@@ -39,24 +51,31 @@ func New(cfg config.LLMConfig) *Rewriter {
if timeout <= 0 {
timeout = 3 * time.Second
}
+ maxTokens := cfg.MaxTokens
+ if maxTokens <= 0 {
+ maxTokens = 512
+ }
+ prompt := strings.TrimSpace(cfg.SystemPrompt)
+ if prompt == "" {
+ prompt = strings.TrimSpace(defaultSystemPrompt)
+ }
return &Rewriter{
- endpoint: base,
- token: cfg.Token,
- model: cfg.Model,
- http: &http.Client{Timeout: timeout},
+ endpoint: base,
+ token: cfg.Token,
+ model: cfg.Model,
+ systemPrompt: prompt,
+ maxTokens: maxTokens,
+ disableThinking: cfg.DisableThinking,
+ http: &http.Client{Timeout: timeout},
}
}
-const systemPrompt = `Ты — нормализатор адресов. Тебе дают адрес, написанный как угодно.
-Верни ТОЛЬКО JSON вида {"variants":["...","..."]} — до 3 вариантов записи того же адреса
-в формате "<название улицы> <дом>[/<дробь>][ <литера>][ к<корпус>][ стр.<строение>]".
-Не выдумывай улицы и дома, которых нет во входе. Если не уверен — верни пустой список.`
-
type chatRequest struct {
- Model string `json:"model"`
- Messages []chatMessage `json:"messages"`
- Temperature float64 `json:"temperature"`
- MaxTokens int `json:"max_tokens"`
+ Model string `json:"model"`
+ Messages []chatMessage `json:"messages"`
+ Temperature float64 `json:"temperature"`
+ MaxTokens int `json:"max_tokens"`
+ ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"`
}
type chatMessage struct {
@@ -72,15 +91,20 @@ type chatResponse struct {
// Rewrite returns up to three normalised variants of raw.
func (r *Rewriter) Rewrite(ctx context.Context, raw string) ([]string, error) {
- payload, err := json.Marshal(chatRequest{
+ reqBody := chatRequest{
Model: r.model,
Messages: []chatMessage{
- {Role: "system", Content: systemPrompt},
+ {Role: "system", Content: r.systemPrompt},
{Role: "user", Content: raw},
},
Temperature: 0,
- MaxTokens: 256,
- })
+ MaxTokens: r.maxTokens,
+ }
+ if r.disableThinking {
+ // Qwen3 and friends: skip chain-of-thought, we only need the JSON.
+ reqBody.ChatTemplateKwargs = map[string]any{"enable_thinking": false}
+ }
+ payload, err := json.Marshal(reqBody)
if err != nil {
return nil, err
}
@@ -114,17 +138,18 @@ func (r *Rewriter) Rewrite(ctx context.Context, raw string) ([]string, error) {
}
// parseVariants extracts the {"variants":[...]} object from a model reply that
-// may be wrapped in prose or code fences.
+// may be wrapped in reasoning or code fences. Thinking models are asked to
+// skip reasoning, but we tolerate it anyway.
func parseVariants(content string) []string {
- start := strings.Index(content, "{")
- end := strings.LastIndex(content, "}")
- if start < 0 || end <= start {
+ content = stripThinking(content)
+ raw := extractVariantsObject(content)
+ if raw == "" {
return nil
}
var parsed struct {
Variants []string `json:"variants"`
}
- if err := json.Unmarshal([]byte(content[start:end+1]), &parsed); err != nil {
+ if err := json.Unmarshal([]byte(raw), &parsed); err != nil {
return nil
}
out := make([]string, 0, len(parsed.Variants))
@@ -135,3 +160,34 @@ func parseVariants(content string) []string {
}
return out
}
+
+var reThinkBlock = regexp.MustCompile(`(?s).*?`)
+
+// stripThinking removes blocks, including an unterminated one left
+// behind when the token budget is exhausted mid-reasoning.
+func stripThinking(content string) string {
+ content = reThinkBlock.ReplaceAllString(content, "")
+ if i := strings.Index(content, ""); i >= 0 {
+ if j := strings.Index(content[i:], ""); j < 0 {
+ content = content[:i]
+ }
+ }
+ return content
+}
+
+// extractVariantsObject returns the smallest {...} that contains "variants".
+func extractVariantsObject(content string) string {
+ idx := strings.Index(content, `"variants"`)
+ if idx < 0 {
+ return ""
+ }
+ start := strings.LastIndex(content[:idx], "{")
+ if start < 0 {
+ return ""
+ }
+ end := strings.Index(content[idx:], "}")
+ if end < 0 {
+ return ""
+ }
+ return content[start : idx+end+1]
+}
diff --git a/internal/adapters/llm/rewriter_test.go b/internal/adapters/llm/rewriter_test.go
new file mode 100644
index 0000000..9dac5bf
--- /dev/null
+++ b/internal/adapters/llm/rewriter_test.go
@@ -0,0 +1,88 @@
+package llm
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "git.totmin.ru/en2zmax/addrkit/internal/config"
+)
+
+// chatServer returns a test server that echoes a fixed content and captures the
+// request body it received.
+func chatServer(t *testing.T, content string) (*httptest.Server, *chatRequest) {
+ t.Helper()
+ var got chatRequest
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewDecoder(r.Body).Decode(&got)
+ w.Header().Set("Content-Type", "application/json")
+ resp := map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}}
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ t.Cleanup(ts.Close)
+ return ts, &got
+}
+
+func TestRewriteUsesConfiguredPrompt(t *testing.T) {
+ ts, got := chatServer(t, `{"variants":["А"]}`)
+ r := New(config.LLMConfig{URL: ts.URL, Model: "m", SystemPrompt: "CUSTOM PROMPT"})
+
+ out, err := r.Rewrite(context.Background(), "вход")
+ if err != nil {
+ t.Fatalf("Rewrite: %v", err)
+ }
+ if len(got.Messages) == 0 || got.Messages[0].Content != "CUSTOM PROMPT" {
+ t.Errorf("system prompt not applied: %+v", got.Messages)
+ }
+ if len(out) != 1 || out[0] != "А" {
+ t.Errorf("variants = %v", out)
+ }
+}
+
+func TestRewriteFallsBackToEmbeddedPrompt(t *testing.T) {
+ ts, got := chatServer(t, `{"variants":[]}`)
+ r := New(config.LLMConfig{URL: ts.URL, Model: "m"})
+
+ if _, err := r.Rewrite(context.Background(), "x"); err != nil {
+ t.Fatalf("Rewrite: %v", err)
+ }
+ if len(got.Messages) == 0 || !strings.Contains(got.Messages[0].Content, "нормализатор") {
+ t.Errorf("embedded default prompt not used: %q", got.Messages[0].Content)
+ }
+}
+
+func TestRewriteDisablesThinking(t *testing.T) {
+ ts, got := chatServer(t, `{"variants":[]}`)
+ r := New(config.LLMConfig{URL: ts.URL, Model: "m", DisableThinking: true})
+
+ if _, err := r.Rewrite(context.Background(), "x"); err != nil {
+ t.Fatalf("Rewrite: %v", err)
+ }
+ if v, ok := got.ChatTemplateKwargs["enable_thinking"]; !ok || v != false {
+ t.Errorf("enable_thinking not disabled: %+v", got.ChatTemplateKwargs)
+ }
+}
+
+func TestParseVariants(t *testing.T) {
+ cases := []struct {
+ name string
+ content string
+ want int
+ }{
+ {"clean", `{"variants":["a","b"]}`, 2},
+ {"with thinking", "hmm\n{\"variants\":[\"a\"]}", 1},
+ {"unterminated thinking", "hmm {\"variants\":[\"a\"]}", 0},
+ {"prose around", "Here you go: {\"variants\":[\"a\",\"b\",\"c\"]} done", 3},
+ {"garbage", "no json here", 0},
+ }
+ for _, tt := range cases {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := parseVariants(tt.content); len(got) != tt.want {
+ t.Errorf("parseVariants(%q) = %v, want %d items", tt.content, got, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index cc8d5e4..3cb6d41 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -46,6 +46,18 @@ type LLMConfig struct {
Token string `json:"token"`
Model string `json:"model"`
TimeoutSeconds int `json:"timeout_seconds"`
+ // MaxTokens bounds the rewriter's reply. Thinking models need enough room
+ // for the final JSON even when thinking is disabled.
+ MaxTokens int `json:"max_tokens"`
+ // DisableThinking turns off chain-of-thought for models that support it
+ // (Qwen3 via chat_template_kwargs). A rewriter only needs the JSON answer.
+ DisableThinking bool `json:"disable_thinking"`
+ // SystemPrompt overrides the built-in rewriter prompt inline. When empty,
+ // the prompt embedded in the binary is used.
+ SystemPrompt string `json:"system_prompt"`
+ // SystemPromptFile loads the rewriter prompt from a file. When set it takes
+ // precedence over SystemPrompt, which keeps long prompts out of the config.
+ SystemPromptFile string `json:"system_prompt_file"`
}
// Default returns a config with sane defaults for the non-required fields.
@@ -63,7 +75,9 @@ func Default() Config {
Retries: 1,
},
LLM: LLMConfig{
- TimeoutSeconds: 3,
+ TimeoutSeconds: 3,
+ MaxTokens: 512,
+ DisableThinking: true,
},
}
}
@@ -79,6 +93,13 @@ func Load(path string) (Config, error) {
if err := dec.Decode(&cfg); err != nil {
return Config{}, fmt.Errorf("parse config: %w", err)
}
+ if cfg.LLM.SystemPromptFile != "" {
+ data, err := os.ReadFile(cfg.LLM.SystemPromptFile)
+ if err != nil {
+ return Config{}, fmt.Errorf("read llm.system_prompt_file: %w", err)
+ }
+ cfg.LLM.SystemPrompt = string(data)
+ }
if err := cfg.validate(); err != nil {
return Config{}, err
}
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
new file mode 100644
index 0000000..1a0cda4
--- /dev/null
+++ b/internal/config/config_test.go
@@ -0,0 +1,69 @@
+package config
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func writeFile(t *testing.T, dir, name, content string) string {
+ t.Helper()
+ path := filepath.Join(dir, name)
+ if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
+ t.Fatalf("write %s: %v", name, err)
+ }
+ return path
+}
+
+func baseConfig(promptFields string) string {
+ return `{
+ "addr": ":6171",
+ "data_path": "registry.db",
+ "portal": { "url": "http://portal" },
+ "llm": { "enabled": true, "url": "http://llm"` + promptFields + ` }
+ }`
+}
+
+func TestLoadInlineSystemPrompt(t *testing.T) {
+ dir := t.TempDir()
+ path := writeFile(t, dir, "config.json", baseConfig(`, "system_prompt": "INLINE PROMPT"`))
+
+ cfg, err := Load(path)
+ if err != nil {
+ t.Fatalf("Load: %v", err)
+ }
+ if cfg.LLM.SystemPrompt != "INLINE PROMPT" {
+ t.Errorf("SystemPrompt = %q", cfg.LLM.SystemPrompt)
+ }
+}
+
+func TestLoadSystemPromptFromFile(t *testing.T) {
+ dir := t.TempDir()
+ promptFile := writeFile(t, dir, "prompt.txt", "PROMPT FROM FILE\n")
+ path := writeFile(t, dir, "config.json", baseConfig(`, "system_prompt_file": "`+promptFile+`"`))
+
+ cfg, err := Load(path)
+ if err != nil {
+ t.Fatalf("Load: %v", err)
+ }
+ if cfg.LLM.SystemPrompt != "PROMPT FROM FILE\n" {
+ t.Errorf("SystemPrompt = %q", cfg.LLM.SystemPrompt)
+ }
+}
+
+func TestLoadMissingSystemPromptFileFailsClosed(t *testing.T) {
+ dir := t.TempDir()
+ path := writeFile(t, dir, "config.json", baseConfig(`, "system_prompt_file": "`+filepath.Join(dir, "nope.txt")+`"`))
+
+ if _, err := Load(path); err == nil {
+ t.Fatal("expected error for missing system_prompt_file")
+ }
+}
+
+func TestLoadRequiresPortalURL(t *testing.T) {
+ dir := t.TempDir()
+ path := writeFile(t, dir, "config.json", `{"addr": ":6171", "data_path": "x.db"}`)
+ if _, err := Load(path); err == nil {
+ t.Fatal("expected error for missing portal.url")
+ }
+}
diff --git a/internal/domain/address/parse.go b/internal/domain/address/parse.go
index f86f0c3..bf4e3a7 100644
--- a/internal/domain/address/parse.go
+++ b/internal/domain/address/parse.go
@@ -120,13 +120,16 @@ func parseStreet(p *ParsedAddress, s string) {
p.StreetKey = streetKey(p.Ordinal, p.NameKey)
}
-// extractOrdinal recognises a leading ordinal ("2-я", "2 я", "вторая") and
-// removes it from the name tokens.
+// extractOrdinal recognises an ordinal and removes it from the name tokens.
+// It handles both the usual leading form ("2-я Сухарная") and an ordinal that
+// appears after the name ("Сухарная 2-я улица"), which both people and models
+// produce.
func extractOrdinal(p *ParsedAddress, tokens []string) []string {
if len(tokens) == 0 {
return tokens
}
+ // Leading forms first: they are by far the most common.
if m := reOrdinalToken.FindStringSubmatch(tokens[0]); m != nil {
p.Ordinal = normalizeOrdinal(m[1])
return tokens[1:]
@@ -139,9 +142,29 @@ func extractOrdinal(p *ParsedAddress, tokens []string) []string {
p.Ordinal = normalizeOrdinal(d)
return tokens[1:]
}
+
+ // Otherwise look for an ordinal anywhere among the name tokens.
+ for i, tok := range tokens {
+ if m := reOrdinalToken.FindStringSubmatch(tok); m != nil {
+ p.Ordinal = normalizeOrdinal(m[1])
+ return removeRange(tokens, i, i+1)
+ }
+ if isDigits(tok) && i+1 < len(tokens) && ordinalSuffixes[tokens[i+1]] {
+ p.Ordinal = normalizeOrdinal(tok)
+ return removeRange(tokens, i, i+2)
+ }
+ }
return tokens
}
+// removeRange returns a copy of tokens with [from, to) removed.
+func removeRange(tokens []string, from, to int) []string {
+ out := make([]string, 0, len(tokens)-(to-from))
+ out = append(out, tokens[:from]...)
+ out = append(out, tokens[to:]...)
+ return out
+}
+
// matchSuffix returns the capture groups of the match of re that ends exactly
// at the end of s, together with the byte offset where it starts. Go's regexp
// is leftmost-first, so we scan all matches and pick the right-anchored one.
diff --git a/internal/domain/match/index.go b/internal/domain/match/index.go
index 16af6f2..97a5ff8 100644
--- a/internal/domain/match/index.go
+++ b/internal/domain/match/index.go
@@ -182,9 +182,9 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
best := 0.0
bestKey := ""
for _, s := range ix.streets {
- // If both sides carry an ordinal and they differ, the streets are
- // different even if the names rhyme.
- if p.Ordinal != "" && s.ordinal != "" && p.Ordinal != s.ordinal {
+ // If the query carries an ordinal, the candidate's ordinal must match:
+ // a user who says "1-й Северный" does not mean the plain "Северная".
+ if !ordinalCompatible(p.Ordinal, s.ordinal) {
continue
}
if sim := nameSimilarity(p.NameKey, s.nameKey); sim > best {
@@ -197,7 +197,7 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
}
for _, s := range ix.streets {
- if p.Ordinal != "" && s.ordinal != "" && p.Ordinal != s.ordinal {
+ if !ordinalCompatible(p.Ordinal, s.ordinal) {
continue
}
if nameSimilarity(p.NameKey, s.nameKey) >= best-ix.opts.AmbiguityMargin {
@@ -210,6 +210,16 @@ func (ix *Index) fuzzyStreets(p address.ParsedAddress) (entries []address.Entry,
return ix.byKey[bestKey], best, nil
}
+// ordinalCompatible reports whether a candidate street may compete with the
+// query given the query's ordinal. When the query specifies an ordinal, only
+// candidates with the same ordinal are allowed; otherwise any candidate is.
+func ordinalCompatible(queryOrdinal, candidateOrdinal string) bool {
+ if queryOrdinal == "" {
+ return true
+ }
+ return queryOrdinal == candidateOrdinal
+}
+
// representatives returns one candidate per competing street, which is far
// more useful to a user than five houses of a single street.
func (ix *Index) representatives(keys []string, score float64) []Candidate {
diff --git a/internal/domain/match/index_test.go b/internal/domain/match/index_test.go
index c173aaa..93d16f1 100644
--- a/internal/domain/match/index_test.go
+++ b/internal/domain/match/index_test.go
@@ -28,6 +28,7 @@ func buildIndex(t *testing.T, addresses ...string) *Index {
func TestResolveStatuses(t *testing.T) {
ix := buildIndex(t,
"Станционная улица 1/1",
+ "2-я Станционная улица 1/1",
"Станционная улица 2/1",
"Станционная улица 3/1 к1",
"Станционная улица 3/1 к2",
@@ -44,7 +45,7 @@ func TestResolveStatuses(t *testing.T) {
name: "found with ordinal, fraction and structure noise",
query: "2-я Станционная улица 1/1 стр.",
wantStatus: address.StatusFound,
- wantAddr: "Станционная улица 1/1",
+ wantAddr: "2-я Станционная улица 1/1",
},
{
name: "found exact house",
@@ -60,9 +61,9 @@ func TestResolveStatuses(t *testing.T) {
},
{
name: "typo in street name tolerated",
- query: "станционая 1/1",
+ query: "искитимсая 5",
wantStatus: address.StatusFound,
- wantAddr: "Станционная улица 1/1",
+ wantAddr: "Искитимская улица 5",
},
{
name: "missing house is partial",
diff --git a/internal/domain/match/levenshtein.go b/internal/domain/match/levenshtein.go
index e6c932c..81d2051 100644
--- a/internal/domain/match/levenshtein.go
+++ b/internal/domain/match/levenshtein.go
@@ -52,6 +52,45 @@ func ratio(a, b string) float64 {
return 1 - float64(levenshtein(a, b))/float64(m)
}
+// damerau returns the optimal string alignment (Damerau-Levenshtein) distance,
+// where an adjacent transposition costs 1. Typos are very often transpositions
+// ("Гоглоя" vs "Гоголя"), so counting them as a single edit matters.
+func damerau(a, b string) int {
+ ra, rb := []rune(a), []rune(b)
+ la, lb := len(ra), len(rb)
+ if la == 0 {
+ return lb
+ }
+ if lb == 0 {
+ return la
+ }
+ // Three rolling rows: d[i-2], d[i-1], d[i].
+ prev2 := make([]int, lb+1)
+ prev := make([]int, lb+1)
+ curr := make([]int, lb+1)
+ for j := 0; j <= lb; j++ {
+ prev[j] = j
+ }
+ for i := 1; i <= la; i++ {
+ curr[0] = i
+ for j := 1; j <= lb; j++ {
+ cost := 1
+ if ra[i-1] == rb[j-1] {
+ cost = 0
+ }
+ best := min3(prev[j]+1, curr[j-1]+1, prev[j-1]+cost)
+ if i > 1 && j > 1 && ra[i-1] == rb[j-2] && ra[i-2] == rb[j-1] {
+ if t := prev2[j-2] + 1; t < best {
+ best = t
+ }
+ }
+ curr[j] = best
+ }
+ prev2, prev, curr = prev, curr, prev2
+ }
+ return prev[lb]
+}
+
func min3(a, b, c int) int {
m := a
if b < m {
diff --git a/internal/domain/match/resolver.go b/internal/domain/match/resolver.go
index 13b670a..17f0747 100644
--- a/internal/domain/match/resolver.go
+++ b/internal/domain/match/resolver.go
@@ -2,6 +2,7 @@ package match
import (
"context"
+ "strings"
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
)
@@ -9,8 +10,9 @@ import (
// Rewriter is an optional, pluggable query rewriter. Its only job is to
// propose alternative surface forms of a user address (for example an LLM
// normalising "кошурникова одиннадцать дробь один" to "Кошевого 11/1"). The
-// rewriter never decides the answer: every proposal is parsed and resolved
-// deterministically, and only entries that exist in the index can be returned.
+// rewriter never decides the answer: every proposal is parsed, validated and
+// resolved deterministically, and only entries that exist in the index can be
+// returned.
//
// It is defined here (consumer side) so the domain stays free of any transport
// or vendor dependency.
@@ -22,7 +24,8 @@ type Rewriter interface {
//
// The deterministic path is always tried first. The rewriter is consulted only
// when the deterministic result is not a confident "found", and its
-// suggestions can only ever turn a miss into a match against real data.
+// suggestions must pass compatibleVariant before being considered, so the
+// model cannot substitute a different street or corrupt the house unit.
type Resolver struct {
index *Index
rewriter Rewriter
@@ -58,7 +61,12 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
best := res
for _, alt := range alternatives {
- candidate := r.index.Resolve(address.Parse(alt))
+ variant := address.Parse(alt)
+ // The model may fix the spelling of the street, nothing else.
+ if !compatibleVariant(parsed, variant) {
+ continue
+ }
+ candidate := r.index.Resolve(variant)
if better(candidate, best) {
best = candidate
}
@@ -66,6 +74,64 @@ func (r *Resolver) Resolve(ctx context.Context, raw string) Result {
return best
}
+// compatibleVariant reports whether an LLM rewrite may be considered. A rewrite
+// is acceptable only when it:
+// - keeps the exact house unit (number, fraction, letter, corpus, structure
+// ignored) — so an apartment is never turned into a corpus;
+// - keeps the ordinal — so "2-я Сухарная" is not collapsed to "Сухарная";
+// - has the same words in the street name, each within a small edit distance
+// — so a different street can never be substituted.
+func compatibleVariant(orig, variant address.ParsedAddress) bool {
+ if !sameHouseUnit(orig, variant) {
+ return false
+ }
+ if orig.Ordinal != variant.Ordinal {
+ return false
+ }
+ to := strings.Fields(orig.NameKey)
+ tv := strings.Fields(variant.NameKey)
+ if len(to) != len(tv) {
+ return false
+ }
+ // NameKey is sorted, so tokens align one-to-one.
+ for i := range to {
+ if !nearToken(to[i], tv[i]) {
+ return false
+ }
+ }
+ return true
+}
+
+// nearToken allows a small number of edits (including transpositions) between
+// two name tokens, scaled to their length.
+func nearToken(a, b string) bool {
+ if a == b {
+ return true
+ }
+ minLen := len([]rune(a))
+ if lb := len([]rune(b)); lb < minLen {
+ minLen = lb
+ }
+ d := damerau(a, b)
+ switch {
+ case d <= 1 && minLen >= 4:
+ return true
+ case d <= 2 && minLen >= 8:
+ return true
+ default:
+ return false
+ }
+}
+
+func sameHouseUnit(a, b address.ParsedAddress) bool {
+ return a.House == b.House &&
+ a.HouseFrac == b.HouseFrac &&
+ a.HouseLetter == b.HouseLetter &&
+ a.Building == b.Building &&
+ a.BuildingFrac == b.BuildingFrac &&
+ a.BuildingLetter == b.BuildingLetter
+}
+
// better ranks results: a stronger status wins, then higher confidence.
func better(a, b Result) bool {
ra, rb := statusRank(a.Status), statusRank(b.Status)
diff --git a/internal/domain/match/resolver_test.go b/internal/domain/match/resolver_test.go
new file mode 100644
index 0000000..c623f26
--- /dev/null
+++ b/internal/domain/match/resolver_test.go
@@ -0,0 +1,52 @@
+package match
+
+import (
+ "testing"
+
+ "git.totmin.ru/en2zmax/addrkit/internal/domain/address"
+)
+
+// TestCompatibleVariant encodes the safety contract for LLM rewrites: only a
+// spelling fix of the same street, with the same ordinal and house unit, may be
+// considered.
+func TestCompatibleVariant(t *testing.T) {
+ orig := address.Parse("2-я Сухарная улица 95")
+ tests := []struct {
+ name string
+ variant string
+ want bool
+ }{
+ {"spelling fix (transposition)", "2-я Сухраная улица 95", true},
+ {"reordered words", "Сухарная 2-я улица 95", true},
+ {"ordinal dropped", "Сухарная улица 95", false},
+ {"different street", "Королева улица 95", false},
+ {"house changed", "2-я Сухарная улица 96", false},
+ {"building added", "2-я Сухарная улица 95 к1", false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := compatibleVariant(orig, address.Parse(tt.variant))
+ if got != tt.want {
+ t.Errorf("compatibleVariant(%q) = %v, want %v", tt.variant, got, tt.want)
+ }
+ })
+ }
+}
+
+// TestCompatibleVariantProtectsApartment ensures "кв 5" is not folded into a
+// corpus "к5" by the model.
+func TestCompatibleVariantProtectsApartment(t *testing.T) {
+ orig := address.Parse("ул. Елоховская, д. 43 кв. 5")
+ if compatibleVariant(orig, address.Parse("ул. Елоховская 43 к5")) {
+ t.Fatal("apartment must not be rewritten as a corpus")
+ }
+}
+
+func TestDamerau(t *testing.T) {
+ if got := damerau("гоглоя", "гоголя"); got != 1 {
+ t.Errorf("transposition distance = %d, want 1", got)
+ }
+ if got := damerau("сееврный", "северная"); got < 2 {
+ t.Errorf("expected a larger distance for a different word, got %d", got)
+ }
+}
diff --git a/internal/eval/eval.go b/internal/eval/eval.go
index e3f6e1a..6b13f97 100644
--- a/internal/eval/eval.go
+++ b/internal/eval/eval.go
@@ -8,6 +8,7 @@
package eval
import (
+ "context"
"fmt"
"math/rand"
"sort"
@@ -55,15 +56,15 @@ type Report struct {
Categories map[string]CategoryStat `json:"categories"`
}
-// Run evaluates all cases against the index.
-func Run(ix *match.Index, cases []Case) Report {
+// Run evaluates all cases against the resolver.
+func Run(resolver *match.Resolver, cases []Case) Report {
rep := Report{Categories: map[string]CategoryStat{}}
for _, c := range cases {
st := rep.Categories[c.Category]
st.Total++
rep.Total++
- res := ix.Resolve(address.Parse(c.Query))
+ res := resolver.Resolve(context.Background(), c.Query)
exp := address.Parse(c.Expect)
exact := false