addrkit: strict ordinal matching, LLM-variant guard, externalized prompt

- match: require candidate ordinal to equal the query ordinal in fuzzy search
  (deterministic typo recall 73% -> 80%, no wrong houses)
- resolver: compatibleVariant guard — LLM may only fix street spelling, keeping
  the same words, ordinal and house unit; damerau distance for transpositions
- llm: prompt moved out of code (go:embed default_prompt.txt) with
  llm.system_prompt / llm.system_prompt_file overrides; guided JSON parsing
  tolerant of thinking blocks
- address: recognize ordinals appearing after the street name
- tests: first coverage for the LLM adapter and config prompt loading; README

Measured (sample 200): deterministic 94.7-95.8% exact / 0 wrong across 4 seeds;
LLM+guard typo 90.7-89.8% / 0 wrong. LLM fallback latency ~1.2-1.9s.
This commit is contained in:
Maksim Totmin
2026-10-09 21:40:46 +07:00
parent ba4705fc4f
commit f1537f1d6b
17 changed files with 590 additions and 48 deletions
+90 -2
View File
@@ -10,8 +10,12 @@ import (
"fmt"
"math/rand"
"os"
"strings"
"time"
"git.totmin.ru/en2zmax/addrkit/internal/adapters/llm"
"git.totmin.ru/en2zmax/addrkit/internal/adapters/registrystore"
"git.totmin.ru/en2zmax/addrkit/internal/config"
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
"git.totmin.ru/en2zmax/addrkit/internal/eval"
"git.totmin.ru/en2zmax/addrkit/internal/seed"
@@ -131,6 +135,9 @@ func runVerify(args []string) int {
func runResolve(args []string) int {
fs := flag.NewFlagSet("resolve", flag.ExitOnError)
db := fs.String("db", "", "SQLite database")
llmURL := fs.String("llm-url", "", "enable the LLM rewriter with this base URL")
llmModel := fs.String("llm-model", "qwen3-14b-awq", "LLM model name")
llmToken := fs.String("llm-token", "", "LLM auth token")
_ = fs.Parse(args)
rest := fs.Args()
if *db == "" || len(rest) == 0 {
@@ -142,7 +149,14 @@ func runResolve(args []string) int {
fmt.Fprintf(os.Stderr, "resolve: %v\n", err)
return 1
}
resolver := match.NewResolver(match.NewIndex(version, entries, match.DefaultOptions()), nil)
var rewriter match.Rewriter
if *llmURL != "" {
rewriter = llm.New(config.LLMConfig{
Enabled: true, URL: *llmURL, Model: *llmModel, Token: *llmToken,
TimeoutSeconds: 5, MaxTokens: 512, DisableThinking: true,
})
}
resolver := match.NewResolver(match.NewIndex(version, entries, match.DefaultOptions()), rewriter)
res := resolver.Resolve(context.Background(), rest[0])
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
@@ -152,11 +166,17 @@ func runResolve(args []string) int {
// runEval generates citizen-style perturbations from the seeded registry and
// reports how well the resolver copes. Results are deterministic per -seed.
// When -llm-url is set, the optional LLM rewriter is enabled and its impact is
// measured on the same cases.
func runEval(args []string) int {
fs := flag.NewFlagSet("eval", flag.ExitOnError)
db := fs.String("db", "", "SQLite database")
sample := fs.Int("sample", 500, "number of registry entries to sample")
seedVal := fs.Int64("seed", 1, "random seed")
llmURL := fs.String("llm-url", "", "enable the LLM rewriter with this base URL")
llmModel := fs.String("llm-model", "qwen3-14b-awq", "LLM model name")
llmToken := fs.String("llm-token", "", "LLM auth token")
categories := fs.String("categories", "", "comma-separated category filter (default: all)")
_ = fs.Parse(args)
if *db == "" {
fmt.Fprintln(os.Stderr, "eval: -db is required")
@@ -168,12 +188,80 @@ func runEval(args []string) int {
return 1
}
ix := match.NewIndex("eval", entries, match.DefaultOptions())
var rewriter match.Rewriter
var counter *countingRewriter
if *llmURL != "" {
counter = &countingRewriter{inner: llm.New(config.LLMConfig{
Enabled: true,
URL: *llmURL,
Model: *llmModel,
Token: *llmToken,
TimeoutSeconds: 5,
MaxTokens: 512,
DisableThinking: true,
})}
rewriter = counter
fmt.Fprintf(os.Stderr, "llm rewriter enabled: %s (%s)\n", *llmURL, *llmModel)
}
resolver := match.NewResolver(ix, rewriter)
rng := rand.New(rand.NewSource(*seedVal))
cases := eval.Generate(entries, *sample, rng)
report := eval.Run(ix, cases)
cases = filterCategories(cases, *categories)
fmt.Fprintf(os.Stderr, "cases: %d\n", len(cases))
report := eval.Run(resolver, cases)
if counter != nil {
fmt.Fprintf(os.Stderr, "llm calls: %d, avg latency: %s\n", counter.calls, counter.avg())
}
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
_ = enc.Encode(report)
return 0
}
// filterCategories keeps only the requested categories (all when spec is empty).
func filterCategories(cases []eval.Case, spec string) []eval.Case {
if strings.TrimSpace(spec) == "" {
return cases
}
want := map[string]bool{}
for _, c := range strings.Split(spec, ",") {
if c = strings.TrimSpace(c); c != "" {
want[c] = true
}
}
out := cases[:0:0]
for _, c := range cases {
if want[c.Category] {
out = append(out, c)
}
}
return out
}
// countingRewriter records how often and how long the rewriter is consulted,
// so the cost of the LLM path is visible in benchmarks.
type countingRewriter struct {
inner match.Rewriter
calls int
total time.Duration
}
func (c *countingRewriter) Rewrite(ctx context.Context, raw string) ([]string, error) {
start := time.Now()
out, err := c.inner.Rewrite(ctx, raw)
c.calls++
c.total += time.Since(start)
return out, err
}
func (c *countingRewriter) avg() time.Duration {
if c.calls == 0 {
return 0
}
return c.total / time.Duration(c.calls)
}