addrkit: deterministic address resolution service (Go)
- domain: address normalization/parsing, matching with explicit statuses - adapters: SQLite registry store, ArcGIS outages client, optional LLM rewriter, HTTP API - seed pipeline with duplicate handling and unparsed reporting - CLI (seed/verify/resolve), systemd unit, OpenAPI contract, tests
This commit is contained in:
addrkit
@@ -0,0 +1,52 @@
|
||||
package seed
|
||||
|
||||
import (
|
||||
"math"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestRealRegistryParseRate validates the parser against a real registry dump.
|
||||
// It is skipped unless ADDRKIT_REGISTRY points at one, so it never runs in a
|
||||
// plain `go test ./...` (no large fixture in the repo).
|
||||
//
|
||||
// Run it explicitly:
|
||||
//
|
||||
// ADDRKIT_REGISTRY=/path/to/address_registry.ndjson \
|
||||
// go test ./internal/seed -run TestRealRegistryParseRate -v
|
||||
func TestRealRegistryParseRate(t *testing.T) {
|
||||
path := os.Getenv("ADDRKIT_REGISTRY")
|
||||
if path == "" {
|
||||
t.Skip("ADDRKIT_REGISTRY not set")
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open registry: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
sources, err := ReadBulkNDJSON(f)
|
||||
if err != nil {
|
||||
t.Fatalf("read registry: %v", err)
|
||||
}
|
||||
entries, stats := Build(sources)
|
||||
|
||||
rate := 0.0
|
||||
if stats.Total > 0 {
|
||||
rate = float64(stats.Parsed) / float64(stats.Total) * 100
|
||||
}
|
||||
t.Logf("total=%d parsed=%d unparsed=%d distinct_streets=%d parse_rate=%.2f%%",
|
||||
stats.Total, stats.Parsed, stats.Unparsed, stats.DistinctStreets, rate)
|
||||
if len(stats.UnparsedSample) > 0 {
|
||||
t.Logf("unparsed sample: %v", stats.UnparsedSample)
|
||||
}
|
||||
|
||||
// A well-formed registry should parse almost perfectly. If this drops we
|
||||
// want a loud failure, not a silently degraded index.
|
||||
if stats.Total > 0 && rate < 98.0 {
|
||||
t.Errorf("parse rate %.2f%% below 98%%", math.Round(rate*100)/100)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Error("no entries produced")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
// Package seed turns a registry dump into canonical address entries. It is the
|
||||
// only place that knows how the source data is shaped.
|
||||
//
|
||||
// The same parser used at query time is used here, so the index and the query
|
||||
// never disagree about how an address decomposes.
|
||||
package seed
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
)
|
||||
|
||||
// Source is one raw registry row.
|
||||
type Source struct {
|
||||
ID string
|
||||
Address string
|
||||
District string
|
||||
}
|
||||
|
||||
// Stats summarises a build and, crucially, surfaces rows the parser could not
|
||||
// structure. A non-empty UnparsedCount is a data-quality signal worth alerting
|
||||
// on, not an error.
|
||||
type Stats struct {
|
||||
Total int `json:"total"`
|
||||
Parsed int `json:"parsed"`
|
||||
Unparsed int `json:"unparsed"`
|
||||
Duplicates int `json:"duplicates"`
|
||||
DistinctStreets int `json:"distinct_streets"`
|
||||
UnparsedSample []string `json:"unparsed_sample,omitempty"`
|
||||
}
|
||||
|
||||
// ReadBulkNDJSON reads an Elasticsearch Bulk NDJSON dump: pairs of an action
|
||||
// line ({"index":{"_id":...}}) followed by a source line
|
||||
// ({"address":...,"district_name":...}). Bare source lines (no action line)
|
||||
// are also accepted, which makes plain JSONL usable too.
|
||||
func ReadBulkNDJSON(r io.Reader) ([]Source, error) {
|
||||
sc := bufio.NewScanner(r)
|
||||
// 1 MiB per line is far beyond any realistic address row.
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
|
||||
var (
|
||||
sources []Source
|
||||
pendingID string
|
||||
)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(line), &obj); err != nil {
|
||||
continue // skip malformed lines rather than aborting the whole seed
|
||||
}
|
||||
|
||||
if action, ok := obj["index"]; ok {
|
||||
var a struct {
|
||||
ID string `json:"_id"`
|
||||
}
|
||||
_ = json.Unmarshal(action, &a)
|
||||
pendingID = a.ID
|
||||
continue
|
||||
}
|
||||
|
||||
if addrRaw, ok := obj["address"]; ok {
|
||||
var addr string
|
||||
if err := json.Unmarshal(addrRaw, &addr); err != nil || addr == "" {
|
||||
continue
|
||||
}
|
||||
src := Source{ID: pendingID, Address: addr}
|
||||
if d, ok := obj["district_name"]; ok {
|
||||
_ = json.Unmarshal(d, &src.District)
|
||||
}
|
||||
sources = append(sources, src)
|
||||
pendingID = ""
|
||||
}
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sources, nil
|
||||
}
|
||||
|
||||
// Build parses every source row into a canonical Entry and reports statistics.
|
||||
// Rows without a house number are still indexed for street lookups but counted
|
||||
// as unparsed, since a registry row without a house is suspicious.
|
||||
func Build(sources []Source) ([]address.Entry, Stats) {
|
||||
entries := make([]address.Entry, 0, len(sources))
|
||||
streets := make(map[string]struct{})
|
||||
seen := make(map[string]struct{})
|
||||
stats := Stats{Total: len(sources)}
|
||||
|
||||
for _, src := range sources {
|
||||
p := address.Parse(src.Address)
|
||||
if p.NameKey == "" {
|
||||
stats.Unparsed++
|
||||
appendSample(&stats, src.Address)
|
||||
continue
|
||||
}
|
||||
entry := address.Entry{
|
||||
ID: src.ID,
|
||||
Address: src.Address,
|
||||
District: src.District,
|
||||
Ordinal: p.Ordinal,
|
||||
NameKey: p.NameKey,
|
||||
NameDisplay: p.NameDisplay,
|
||||
StreetType: p.StreetType,
|
||||
StreetKey: p.StreetKey,
|
||||
House: p.House,
|
||||
HouseFrac: p.HouseFrac,
|
||||
HouseLetter: p.HouseLetter,
|
||||
Building: p.Building,
|
||||
BuildingFrac: p.BuildingFrac,
|
||||
BuildingLetter: p.BuildingLetter,
|
||||
Structure: p.Structure,
|
||||
}
|
||||
|
||||
// The source registry contains rows that differ only by "стр."
|
||||
// (structure). Since structure does not affect which house an outage
|
||||
// belongs to, drop such duplicates rather than surfacing them as
|
||||
// spurious ambiguity at query time.
|
||||
if key := dedupKey(entry); key != "" {
|
||||
if _, ok := seen[key]; ok {
|
||||
stats.Duplicates++
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
|
||||
entries = append(entries, entry)
|
||||
streets[p.StreetKey] = struct{}{}
|
||||
if p.HasHouse {
|
||||
stats.Parsed++
|
||||
} else {
|
||||
stats.Unparsed++
|
||||
appendSample(&stats, src.Address)
|
||||
}
|
||||
}
|
||||
stats.DistinctStreets = len(streets)
|
||||
sort.Strings(stats.UnparsedSample)
|
||||
return entries, stats
|
||||
}
|
||||
|
||||
// dedupKey identifies the same house unit within the same street, ignoring the
|
||||
// structure component.
|
||||
func dedupKey(e address.Entry) string {
|
||||
if e.House == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.Join([]string{
|
||||
e.StreetKey, e.House, e.HouseFrac, e.HouseLetter,
|
||||
e.Building, e.BuildingFrac, e.BuildingLetter,
|
||||
}, "|")
|
||||
}
|
||||
|
||||
func appendSample(s *Stats, address string) {
|
||||
const maxSamples = 20
|
||||
if len(s.UnparsedSample) < maxSamples {
|
||||
s.UnparsedSample = append(s.UnparsedSample, address)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package seed
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReadBulkNDJSON(t *testing.T) {
|
||||
input := `{"index": {"_index": "address_registry", "_id": "a1"}}
|
||||
{"address": "Станционная улица 1/1", "district_name": "Ленинский"}
|
||||
{"index": {"_id": "a2"}}
|
||||
{"address": "Искитимская улица 5", "district_name": "Кировский"}
|
||||
|
||||
{"address": "Третья улица 7"}
|
||||
`
|
||||
src, err := ReadBulkNDJSON(strings.NewReader(input))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadBulkNDJSON: %v", err)
|
||||
}
|
||||
if len(src) != 3 {
|
||||
t.Fatalf("got %d sources, want 3", len(src))
|
||||
}
|
||||
if src[0].ID != "a1" || src[0].District != "Ленинский" {
|
||||
t.Errorf("first source = %+v", src[0])
|
||||
}
|
||||
if src[2].Address != "Третья улица 7" {
|
||||
t.Errorf("bare source not read: %+v", src[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuild(t *testing.T) {
|
||||
sources := []Source{
|
||||
{ID: "1", Address: "Станционная улица 1/1"},
|
||||
{ID: "2", Address: "Искитимская улица 5"},
|
||||
{ID: "3", Address: "Станционная улица"}, // no house -> unparsed
|
||||
}
|
||||
entries, stats := Build(sources)
|
||||
|
||||
if stats.Total != 3 {
|
||||
t.Errorf("Total = %d, want 3", stats.Total)
|
||||
}
|
||||
if stats.Parsed != 2 {
|
||||
t.Errorf("Parsed = %d, want 2", stats.Parsed)
|
||||
}
|
||||
if stats.Unparsed != 1 {
|
||||
t.Errorf("Unparsed = %d, want 1", stats.Unparsed)
|
||||
}
|
||||
if len(entries) != 3 {
|
||||
t.Errorf("entries = %d, want 3", len(entries))
|
||||
}
|
||||
// Two distinct streets: "станционная" and "искитимская".
|
||||
if stats.DistinctStreets != 2 {
|
||||
t.Errorf("DistinctStreets = %d, want 2", stats.DistinctStreets)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user