Maksim Totmin
17ff5c591f
- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
53 lines
1.4 KiB
Go
53 lines
1.4 KiB
Go
package seed
|
|
|
|
import (
|
|
"math"
|
|
"os"
|
|
"testing"
|
|
)
|
|
|
|
// TestRealRegistryParseRate validates the parser against a real registry dump.
|
|
// It is skipped unless ADDRKIT_REGISTRY points at one, so it never runs in a
|
|
// plain `go test ./...` (no large fixture in the repo).
|
|
//
|
|
// Run it explicitly:
|
|
//
|
|
// ADDRKIT_REGISTRY=/path/to/address_registry.ndjson \
|
|
// go test ./internal/seed -run TestRealRegistryParseRate -v
|
|
func TestRealRegistryParseRate(t *testing.T) {
|
|
path := os.Getenv("ADDRKIT_REGISTRY")
|
|
if path == "" {
|
|
t.Skip("ADDRKIT_REGISTRY not set")
|
|
}
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
t.Fatalf("open registry: %v", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
sources, err := ReadBulkNDJSON(f)
|
|
if err != nil {
|
|
t.Fatalf("read registry: %v", err)
|
|
}
|
|
entries, stats := Build(sources, BuildOptions{})
|
|
|
|
rate := 0.0
|
|
if stats.Total > 0 {
|
|
rate = float64(stats.Parsed) / float64(stats.Total) * 100
|
|
}
|
|
t.Logf("total=%d parsed=%d unparsed=%d distinct_streets=%d parse_rate=%.2f%%",
|
|
stats.Total, stats.Parsed, stats.Unparsed, stats.DistinctStreets, rate)
|
|
if len(stats.UnparsedSample) > 0 {
|
|
t.Logf("unparsed sample: %v", stats.UnparsedSample)
|
|
}
|
|
|
|
// A well-formed registry should parse almost perfectly. If this drops we
|
|
// want a loud failure, not a silently degraded index.
|
|
if stats.Total > 0 && rate < 98.0 {
|
|
t.Errorf("parse rate %.2f%% below 98%%", math.Round(rate*100)/100)
|
|
}
|
|
if len(entries) == 0 {
|
|
t.Error("no entries produced")
|
|
}
|
|
}
|