refactor!: make addrkit a multi-city address normalizer
- remove the events/outages module, ArcGIS adapter, portal config and event statuses - add a configurable city dictionary/parser and make City part of StreetKey - seed: read city/city_name, namespace IDs by city, keep structures distinct - fix reliability: schema v2 with fail-closed load, deterministic ordering, input limits, config validation, atomic DB publish, LLM response caps - add read-only shadow mode comparing the old ES path with addrkit - breaking: registry.db must be re-seeded; config has no portal section
This commit is contained in:
Maksim Totmin
@@ -5,37 +5,49 @@ package registrystore
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
_ "modernc.org/sqlite" // registers the "sqlite" driver
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
)
|
||||
|
||||
// SchemaVersion identifies the on-disk format. It is independent of the data
|
||||
// version (the registry snapshot date) and must be bumped whenever the key
|
||||
// format or columns change. A service refuses to open a database it does not
|
||||
// understand instead of silently resolving against half-empty fields.
|
||||
const SchemaVersion = "2"
|
||||
|
||||
const schema = `
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS address (
|
||||
id TEXT PRIMARY KEY,
|
||||
address TEXT NOT NULL,
|
||||
district TEXT NOT NULL DEFAULT '',
|
||||
ordinal TEXT NOT NULL DEFAULT '',
|
||||
name_key TEXT NOT NULL DEFAULT '',
|
||||
name_display TEXT NOT NULL DEFAULT '',
|
||||
street_type TEXT NOT NULL DEFAULT '',
|
||||
street_key TEXT NOT NULL,
|
||||
house TEXT NOT NULL DEFAULT '',
|
||||
house_frac TEXT NOT NULL DEFAULT '',
|
||||
house_letter TEXT NOT NULL DEFAULT '',
|
||||
building TEXT NOT NULL DEFAULT '',
|
||||
building_frac TEXT NOT NULL DEFAULT '',
|
||||
id TEXT PRIMARY KEY,
|
||||
address TEXT NOT NULL,
|
||||
district TEXT NOT NULL DEFAULT '',
|
||||
city TEXT NOT NULL DEFAULT '',
|
||||
city_key TEXT NOT NULL DEFAULT '',
|
||||
ordinal TEXT NOT NULL DEFAULT '',
|
||||
name_key TEXT NOT NULL DEFAULT '',
|
||||
name_display TEXT NOT NULL DEFAULT '',
|
||||
street_type TEXT NOT NULL DEFAULT '',
|
||||
street_key TEXT NOT NULL,
|
||||
house TEXT NOT NULL DEFAULT '',
|
||||
house_frac TEXT NOT NULL DEFAULT '',
|
||||
house_letter TEXT NOT NULL DEFAULT '',
|
||||
building TEXT NOT NULL DEFAULT '',
|
||||
building_frac TEXT NOT NULL DEFAULT '',
|
||||
building_letter TEXT NOT NULL DEFAULT '',
|
||||
structure TEXT NOT NULL DEFAULT ''
|
||||
structure TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_address_street ON address(street_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_address_city ON address(city_key);
|
||||
`
|
||||
|
||||
// Save writes entries to path atomically: it builds a temporary file and
|
||||
@@ -44,13 +56,21 @@ CREATE INDEX IF NOT EXISTS idx_address_street ON address(street_key);
|
||||
// The build uses the rollback journal (not WAL) on purpose, so that the result
|
||||
// is a single file with no companion -wal/-shm that a rename would orphan.
|
||||
func Save(path, version string, entries []address.Entry) error {
|
||||
tmp := path + ".tmp"
|
||||
removeDBFiles(tmp)
|
||||
if len(entries) == 0 {
|
||||
return errors.New("refusing to publish an empty index")
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite", "file:"+tmp+
|
||||
"?_pragma=journal_mode(DELETE)"+
|
||||
"&_pragma=synchronous(FULL)"+
|
||||
"&_pragma=busy_timeout(5000)")
|
||||
tmp := path + ".tmp"
|
||||
// A previous failed build may have left a complete temp database with the
|
||||
// same meta rows; reusing it would hit a primary-key conflict.
|
||||
_ = os.Remove(tmp)
|
||||
removeSidecars(tmp)
|
||||
|
||||
uri, err := dsn(tmp, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db, err := sql.Open("sqlite", uri)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open temp db: %w", err)
|
||||
}
|
||||
@@ -68,12 +88,46 @@ func Save(path, version string, entries []address.Entry) error {
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
return fmt.Errorf("publish db: %w", err)
|
||||
}
|
||||
// Drop only the temp file's journal siblings; the published base file must
|
||||
// survive.
|
||||
// The rename is atomic, but a crash immediately after it may lose the
|
||||
// directory entry. Syncing the parent makes the publish durable.
|
||||
syncDir(filepath.Dir(path))
|
||||
removeSidecars(tmp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// dsn builds a SQLite URI. Concatenating "file:" + path breaks on paths that
|
||||
// contain "?", "#" or "%", so let net/url escape the path for us.
|
||||
func dsn(path string, readOnly bool) (string, error) {
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve db path: %w", err)
|
||||
}
|
||||
u := url.URL{Scheme: "file", Path: abs}
|
||||
q := u.Query()
|
||||
q.Add("_pragma", "busy_timeout(5000)")
|
||||
if readOnly {
|
||||
q.Set("mode", "ro")
|
||||
q.Add("_pragma", "query_only(1)")
|
||||
} else {
|
||||
q.Add("_pragma", "journal_mode(DELETE)")
|
||||
q.Add("_pragma", "synchronous(FULL)")
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
// syncDir fsyncs a directory so a freshly renamed file survives a crash. It is
|
||||
// best effort: some filesystems do not support directory fsync, and failing to
|
||||
// sync here is no worse than not trying.
|
||||
func syncDir(dir string) {
|
||||
f, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
_ = f.Sync()
|
||||
}
|
||||
|
||||
func insertAll(db *sql.DB, version string, entries []address.Entry) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
@@ -84,11 +138,15 @@ func insertAll(db *sql.DB, version string, entries []address.Entry) error {
|
||||
if _, err := tx.Exec(`INSERT INTO meta(key, value) VALUES ('version', ?)`, version); err != nil {
|
||||
return fmt.Errorf("insert meta: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO meta(key, value) VALUES ('schema_version', ?)`, SchemaVersion); err != nil {
|
||||
return fmt.Errorf("insert schema version: %w", err)
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`INSERT INTO address (
|
||||
id, address, district, ordinal, name_key, name_display, street_type, street_key,
|
||||
house, house_frac, house_letter, building, building_frac, building_letter, structure
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
|
||||
id, address, district, city, city_key, ordinal, name_key, name_display,
|
||||
street_type, street_key, house, house_frac, house_letter, building,
|
||||
building_frac, building_letter, structure
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare insert: %w", err)
|
||||
}
|
||||
@@ -101,8 +159,9 @@ func insertAll(db *sql.DB, version string, entries []address.Entry) error {
|
||||
id = fmt.Sprintf("row-%d", i)
|
||||
}
|
||||
if _, err := stmt.Exec(
|
||||
id, e.Address, e.District, e.Ordinal, e.NameKey, e.NameDisplay, e.StreetType, e.StreetKey,
|
||||
e.House, e.HouseFrac, e.HouseLetter, e.Building, e.BuildingFrac, e.BuildingLetter, e.Structure,
|
||||
id, e.Address, e.District, e.City, e.CityKey, e.Ordinal, e.NameKey, e.NameDisplay,
|
||||
e.StreetType, e.StreetKey, e.House, e.HouseFrac, e.HouseLetter, e.Building,
|
||||
e.BuildingFrac, e.BuildingLetter, e.Structure,
|
||||
); err != nil {
|
||||
return fmt.Errorf("insert %q: %w", e.Address, err)
|
||||
}
|
||||
@@ -114,26 +173,41 @@ func insertAll(db *sql.DB, version string, entries []address.Entry) error {
|
||||
}
|
||||
|
||||
// Load opens path read-only and returns the data version and all entries.
|
||||
// It fails closed when the file was produced by an incompatible schema.
|
||||
func Load(path string) (version string, entries []address.Entry, err error) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return "", nil, fmt.Errorf("stat db: %w", err)
|
||||
}
|
||||
db, err := sql.Open("sqlite", "file:"+path+
|
||||
"?mode=ro"+
|
||||
"&_pragma=query_only(1)"+
|
||||
"&_pragma=busy_timeout(5000)")
|
||||
uri, err := dsn(path, true)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
db, err := sql.Open("sqlite", uri)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("open db: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
var schemaVersion string
|
||||
if err := db.QueryRow(`SELECT value FROM meta WHERE key='schema_version'`).Scan(&schemaVersion); err != nil {
|
||||
return "", nil, fmt.Errorf("read schema version (re-run addrkit seed): %w", err)
|
||||
}
|
||||
if schemaVersion != SchemaVersion {
|
||||
return "", nil, fmt.Errorf(
|
||||
"unsupported index schema %q (want %q); re-run addrkit seed",
|
||||
schemaVersion, SchemaVersion,
|
||||
)
|
||||
}
|
||||
if err := db.QueryRow(`SELECT value FROM meta WHERE key='version'`).Scan(&version); err != nil {
|
||||
return "", nil, fmt.Errorf("read version: %w", err)
|
||||
}
|
||||
|
||||
rows, err := db.Query(`SELECT id, address, district, ordinal, name_key, name_display,
|
||||
street_type, street_key, house, house_frac, house_letter, building, building_frac,
|
||||
building_letter, structure FROM address`)
|
||||
rows, err := db.Query(`SELECT id, address, district, city, city_key, ordinal,
|
||||
name_key, name_display, street_type, street_key, house, house_frac,
|
||||
house_letter, building, building_frac, building_letter, structure
|
||||
FROM address
|
||||
ORDER BY city_key, street_key, house, house_frac, house_letter,
|
||||
building, building_frac, building_letter, structure, address, id`)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("query: %w", err)
|
||||
}
|
||||
@@ -142,9 +216,10 @@ func Load(path string) (version string, entries []address.Entry, err error) {
|
||||
for rows.Next() {
|
||||
var e address.Entry
|
||||
if err := rows.Scan(
|
||||
&e.ID, &e.Address, &e.District, &e.Ordinal, &e.NameKey, &e.NameDisplay,
|
||||
&e.StreetType, &e.StreetKey, &e.House, &e.HouseFrac, &e.HouseLetter,
|
||||
&e.Building, &e.BuildingFrac, &e.BuildingLetter, &e.Structure,
|
||||
&e.ID, &e.Address, &e.District, &e.City, &e.CityKey, &e.Ordinal,
|
||||
&e.NameKey, &e.NameDisplay, &e.StreetType, &e.StreetKey, &e.House,
|
||||
&e.HouseFrac, &e.HouseLetter, &e.Building, &e.BuildingFrac,
|
||||
&e.BuildingLetter, &e.Structure,
|
||||
); err != nil {
|
||||
return "", nil, fmt.Errorf("scan: %w", err)
|
||||
}
|
||||
@@ -156,12 +231,6 @@ func Load(path string) (version string, entries []address.Entry, err error) {
|
||||
return version, entries, nil
|
||||
}
|
||||
|
||||
// removeDBFiles deletes a database file and its journal siblings.
|
||||
func removeDBFiles(base string) {
|
||||
_ = os.Remove(base)
|
||||
removeSidecars(base)
|
||||
}
|
||||
|
||||
// removeSidecars deletes only the -wal/-shm companions of a database file.
|
||||
func removeSidecars(base string) {
|
||||
_ = os.Remove(base + "-wal")
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package registrystore
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
|
||||
@@ -11,11 +13,13 @@ func TestSaveLoadRoundTrip(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "registry.db")
|
||||
|
||||
addr := "Станционная улица 1/1"
|
||||
p := address.Parse(addr)
|
||||
parser := address.NewParser(address.NewDictionary([]address.City{{Name: "Новосибирск"}}), "")
|
||||
p := parser.ParseInCity(addr, "Новосибирск")
|
||||
entries := []address.Entry{{
|
||||
ID: "a1", Address: addr, District: "Ленинский",
|
||||
City: p.City, CityKey: p.CityKey,
|
||||
Ordinal: p.Ordinal, NameKey: p.NameKey, StreetType: p.StreetType, StreetKey: p.StreetKey,
|
||||
House: p.House, HouseFrac: p.HouseFrac,
|
||||
House: p.House, HouseFrac: p.HouseFrac, Structure: "2",
|
||||
}}
|
||||
|
||||
if err := Save(path, "2026-10-09", entries); err != nil {
|
||||
@@ -35,6 +39,58 @@ func TestSaveLoadRoundTrip(t *testing.T) {
|
||||
if got[0].Address != addr || got[0].District != "Ленинский" || got[0].StreetKey != p.StreetKey {
|
||||
t.Errorf("round-tripped entry mismatch: %+v", got[0])
|
||||
}
|
||||
if got[0].City != "Новосибирск" || got[0].CityKey != "новосибирск" || got[0].Structure != "2" {
|
||||
t.Errorf("city/structure not round-tripped: %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveRejectsEmptyIndex(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "registry.db")
|
||||
if err := Save(path, "v1", nil); err == nil {
|
||||
t.Fatal("expected error for empty index")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsIncompatibleSchema(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "old.db")
|
||||
uri, err := dsn(path, false)
|
||||
if err != nil {
|
||||
t.Fatalf("dsn: %v", err)
|
||||
}
|
||||
db, err := sql.Open("sqlite", uri)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`); err != nil {
|
||||
t.Fatalf("create meta: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`INSERT INTO meta(key, value) VALUES ('schema_version', '1')`); err != nil {
|
||||
t.Fatalf("insert schema version: %v", err)
|
||||
}
|
||||
db.Close()
|
||||
|
||||
_, _, err = Load(path)
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported index schema") {
|
||||
t.Fatalf("Load error = %v, want unsupported schema", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPathWithSpecialCharacters guards the SQLite URI construction: a naive
|
||||
// "file:" + path breaks on ?, # and % in the path.
|
||||
func TestPathWithSpecialCharacters(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "registry ? # 100%.db")
|
||||
entries := []address.Entry{{ID: "x", Address: "Первая улица 1"}}
|
||||
|
||||
if err := Save(path, "v1", entries); err != nil {
|
||||
t.Fatalf("Save: %v", err)
|
||||
}
|
||||
version, got, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if version != "v1" || len(got) != 1 {
|
||||
t.Fatalf("version=%q entries=%+v", version, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSaveIsAtomicOverwrite verifies that re-seeding replaces the previous data
|
||||
|
||||
Reference in New Issue
Block a user