addrkit: strict ordinal matching, LLM-variant guard, externalized prompt
- match: require candidate ordinal to equal the query ordinal in fuzzy search (deterministic typo recall 73% -> 80%, no wrong houses) - resolver: compatibleVariant guard — LLM may only fix street spelling, keeping the same words, ordinal and house unit; damerau distance for transpositions - llm: prompt moved out of code (go:embed default_prompt.txt) with llm.system_prompt / llm.system_prompt_file overrides; guided JSON parsing tolerant of thinking blocks - address: recognize ordinals appearing after the street name - tests: first coverage for the LLM adapter and config prompt loading; README Measured (sample 200): deterministic 94.7-95.8% exact / 0 wrong across 4 seeds; LLM+guard typo 90.7-89.8% / 0 wrong. LLM fallback latency ~1.2-1.9s.
This commit is contained in:
+90
-2
@@ -10,8 +10,12 @@ import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/adapters/llm"
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/adapters/registrystore"
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/config"
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/eval"
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/seed"
|
||||
@@ -131,6 +135,9 @@ func runVerify(args []string) int {
|
||||
func runResolve(args []string) int {
|
||||
fs := flag.NewFlagSet("resolve", flag.ExitOnError)
|
||||
db := fs.String("db", "", "SQLite database")
|
||||
llmURL := fs.String("llm-url", "", "enable the LLM rewriter with this base URL")
|
||||
llmModel := fs.String("llm-model", "qwen3-14b-awq", "LLM model name")
|
||||
llmToken := fs.String("llm-token", "", "LLM auth token")
|
||||
_ = fs.Parse(args)
|
||||
rest := fs.Args()
|
||||
if *db == "" || len(rest) == 0 {
|
||||
@@ -142,7 +149,14 @@ func runResolve(args []string) int {
|
||||
fmt.Fprintf(os.Stderr, "resolve: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
resolver := match.NewResolver(match.NewIndex(version, entries, match.DefaultOptions()), nil)
|
||||
var rewriter match.Rewriter
|
||||
if *llmURL != "" {
|
||||
rewriter = llm.New(config.LLMConfig{
|
||||
Enabled: true, URL: *llmURL, Model: *llmModel, Token: *llmToken,
|
||||
TimeoutSeconds: 5, MaxTokens: 512, DisableThinking: true,
|
||||
})
|
||||
}
|
||||
resolver := match.NewResolver(match.NewIndex(version, entries, match.DefaultOptions()), rewriter)
|
||||
res := resolver.Resolve(context.Background(), rest[0])
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
@@ -152,11 +166,17 @@ func runResolve(args []string) int {
|
||||
|
||||
// runEval generates citizen-style perturbations from the seeded registry and
|
||||
// reports how well the resolver copes. Results are deterministic per -seed.
|
||||
// When -llm-url is set, the optional LLM rewriter is enabled and its impact is
|
||||
// measured on the same cases.
|
||||
func runEval(args []string) int {
|
||||
fs := flag.NewFlagSet("eval", flag.ExitOnError)
|
||||
db := fs.String("db", "", "SQLite database")
|
||||
sample := fs.Int("sample", 500, "number of registry entries to sample")
|
||||
seedVal := fs.Int64("seed", 1, "random seed")
|
||||
llmURL := fs.String("llm-url", "", "enable the LLM rewriter with this base URL")
|
||||
llmModel := fs.String("llm-model", "qwen3-14b-awq", "LLM model name")
|
||||
llmToken := fs.String("llm-token", "", "LLM auth token")
|
||||
categories := fs.String("categories", "", "comma-separated category filter (default: all)")
|
||||
_ = fs.Parse(args)
|
||||
if *db == "" {
|
||||
fmt.Fprintln(os.Stderr, "eval: -db is required")
|
||||
@@ -168,12 +188,80 @@ func runEval(args []string) int {
|
||||
return 1
|
||||
}
|
||||
ix := match.NewIndex("eval", entries, match.DefaultOptions())
|
||||
|
||||
var rewriter match.Rewriter
|
||||
var counter *countingRewriter
|
||||
if *llmURL != "" {
|
||||
counter = &countingRewriter{inner: llm.New(config.LLMConfig{
|
||||
Enabled: true,
|
||||
URL: *llmURL,
|
||||
Model: *llmModel,
|
||||
Token: *llmToken,
|
||||
TimeoutSeconds: 5,
|
||||
MaxTokens: 512,
|
||||
DisableThinking: true,
|
||||
})}
|
||||
rewriter = counter
|
||||
fmt.Fprintf(os.Stderr, "llm rewriter enabled: %s (%s)\n", *llmURL, *llmModel)
|
||||
}
|
||||
resolver := match.NewResolver(ix, rewriter)
|
||||
|
||||
rng := rand.New(rand.NewSource(*seedVal))
|
||||
cases := eval.Generate(entries, *sample, rng)
|
||||
report := eval.Run(ix, cases)
|
||||
cases = filterCategories(cases, *categories)
|
||||
fmt.Fprintf(os.Stderr, "cases: %d\n", len(cases))
|
||||
|
||||
report := eval.Run(resolver, cases)
|
||||
|
||||
if counter != nil {
|
||||
fmt.Fprintf(os.Stderr, "llm calls: %d, avg latency: %s\n", counter.calls, counter.avg())
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
_ = enc.Encode(report)
|
||||
return 0
|
||||
}
|
||||
|
||||
// filterCategories keeps only the requested categories (all when spec is empty).
|
||||
func filterCategories(cases []eval.Case, spec string) []eval.Case {
|
||||
if strings.TrimSpace(spec) == "" {
|
||||
return cases
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, c := range strings.Split(spec, ",") {
|
||||
if c = strings.TrimSpace(c); c != "" {
|
||||
want[c] = true
|
||||
}
|
||||
}
|
||||
out := cases[:0:0]
|
||||
for _, c := range cases {
|
||||
if want[c.Category] {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// countingRewriter records how often and how long the rewriter is consulted,
|
||||
// so the cost of the LLM path is visible in benchmarks.
|
||||
type countingRewriter struct {
|
||||
inner match.Rewriter
|
||||
calls int
|
||||
total time.Duration
|
||||
}
|
||||
|
||||
func (c *countingRewriter) Rewrite(ctx context.Context, raw string) ([]string, error) {
|
||||
start := time.Now()
|
||||
out, err := c.inner.Rewrite(ctx, raw)
|
||||
c.calls++
|
||||
c.total += time.Since(start)
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (c *countingRewriter) avg() time.Duration {
|
||||
if c.calls == 0 {
|
||||
return 0
|
||||
}
|
||||
return c.total / time.Duration(c.calls)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user