// Package llm implements the optional, disabled-by-default query rewriter. // // It talks to any OpenAI-compatible chat completions endpoint (for example the // on-prem vLLM already used by AMS). Its contract is deliberately narrow: it // may only propose alternative spellings of the user's address. It never // returns the answer, and its output is validated by the deterministic // resolver, so a misbehaving model can at worst waste a request — it cannot // make the service answer with a wrong address. package llm import ( "bytes" "context" _ "embed" "encoding/json" "fmt" "net/http" "regexp" "strings" "time" "git.totmin.ru/en2zmax/addrkit/internal/config" ) // defaultSystemPrompt is embedded from a readable text file so the prompt can // be reviewed and versioned like documentation. It can be overridden per // deployment via llm.system_prompt or llm.system_prompt_file in the config. // //go:embed default_prompt.txt var defaultSystemPrompt string // Rewriter asks a model for normalised variants of an address. type Rewriter struct { endpoint string token string model string systemPrompt string maxTokens int disableThinking bool http *http.Client } // New builds a rewriter from configuration. The URL is treated as a base; the // OpenAI path is appended if it is not already present. func New(cfg config.LLMConfig) *Rewriter { base := strings.TrimRight(cfg.URL, "/") if !strings.HasSuffix(base, "/chat/completions") { base += "/v1/chat/completions" } timeout := time.Duration(cfg.TimeoutSeconds) * time.Second if timeout <= 0 { timeout = 3 * time.Second } maxTokens := cfg.MaxTokens if maxTokens <= 0 { maxTokens = 512 } prompt := strings.TrimSpace(cfg.SystemPrompt) if prompt == "" { prompt = strings.TrimSpace(defaultSystemPrompt) } return &Rewriter{ endpoint: base, token: cfg.Token, model: cfg.Model, systemPrompt: prompt, maxTokens: maxTokens, disableThinking: cfg.DisableThinking, http: &http.Client{Timeout: timeout}, } } type chatRequest struct { Model string `json:"model"` Messages []chatMessage `json:"messages"` Temperature float64 `json:"temperature"` MaxTokens int `json:"max_tokens"` ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"` } type chatMessage struct { Role string `json:"role"` Content string `json:"content"` } type chatResponse struct { Choices []struct { Message chatMessage `json:"message"` } `json:"choices"` } // Rewrite returns up to three normalised variants of raw. func (r *Rewriter) Rewrite(ctx context.Context, raw string) ([]string, error) { reqBody := chatRequest{ Model: r.model, Messages: []chatMessage{ {Role: "system", Content: r.systemPrompt}, {Role: "user", Content: raw}, }, Temperature: 0, MaxTokens: r.maxTokens, } if r.disableThinking { // Qwen3 and friends: skip chain-of-thought, we only need the JSON. reqBody.ChatTemplateKwargs = map[string]any{"enable_thinking": false} } payload, err := json.Marshal(reqBody) if err != nil { return nil, err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.endpoint, bytes.NewReader(payload)) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") if r.token != "" && r.token != "EMPTY" { req.Header.Set("Authorization", "Bearer "+r.token) } resp, err := r.http.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("llm status %d", resp.StatusCode) } var cr chatResponse if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil { return nil, fmt.Errorf("decode llm response: %w", err) } if len(cr.Choices) == 0 { return nil, nil } return parseVariants(cr.Choices[0].Message.Content), nil } // parseVariants extracts the {"variants":[...]} object from a model reply that // may be wrapped in reasoning or code fences. Thinking models are asked to // skip reasoning, but we tolerate it anyway. func parseVariants(content string) []string { content = stripThinking(content) raw := extractVariantsObject(content) if raw == "" { return nil } var parsed struct { Variants []string `json:"variants"` } if err := json.Unmarshal([]byte(raw), &parsed); err != nil { return nil } out := make([]string, 0, len(parsed.Variants)) for _, v := range parsed.Variants { if s := strings.TrimSpace(v); s != "" { out = append(out, s) } } return out } var reThinkBlock = regexp.MustCompile(`(?s).*?`) // stripThinking removes blocks, including an unterminated one left // behind when the token budget is exhausted mid-reasoning. func stripThinking(content string) string { content = reThinkBlock.ReplaceAllString(content, "") if i := strings.Index(content, ""); i >= 0 { if j := strings.Index(content[i:], ""); j < 0 { content = content[:i] } } return content } // extractVariantsObject returns the smallest {...} that contains "variants". func extractVariantsObject(content string) string { idx := strings.Index(content, `"variants"`) if idx < 0 { return "" } start := strings.LastIndex(content[:idx], "{") if start < 0 { return "" } end := strings.Index(content[idx:], "}") if end < 0 { return "" } return content[start : idx+end+1] }