// Package seed turns a registry dump into canonical address entries. It is the // only place that knows how the source data is shaped. // // The same parser used at query time is used here, so the index and the query // never disagree about how an address decomposes. package seed import ( "bufio" "encoding/json" "io" "sort" "strings" "git.totmin.ru/en2zmax/addrkit/internal/domain/address" ) // Source is one raw registry row. type Source struct { ID string Address string District string } // Stats summarises a build and, crucially, surfaces rows the parser could not // structure. A non-empty UnparsedCount is a data-quality signal worth alerting // on, not an error. type Stats struct { Total int `json:"total"` Parsed int `json:"parsed"` Unparsed int `json:"unparsed"` Duplicates int `json:"duplicates"` DistinctStreets int `json:"distinct_streets"` UnparsedSample []string `json:"unparsed_sample,omitempty"` } // ReadBulkNDJSON reads an Elasticsearch Bulk NDJSON dump: pairs of an action // line ({"index":{"_id":...}}) followed by a source line // ({"address":...,"district_name":...}). Bare source lines (no action line) // are also accepted, which makes plain JSONL usable too. func ReadBulkNDJSON(r io.Reader) ([]Source, error) { sc := bufio.NewScanner(r) // 1 MiB per line is far beyond any realistic address row. sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) var ( sources []Source pendingID string ) for sc.Scan() { line := strings.TrimSpace(sc.Text()) if line == "" { continue } var obj map[string]json.RawMessage if err := json.Unmarshal([]byte(line), &obj); err != nil { continue // skip malformed lines rather than aborting the whole seed } if action, ok := obj["index"]; ok { var a struct { ID string `json:"_id"` } _ = json.Unmarshal(action, &a) pendingID = a.ID continue } if addrRaw, ok := obj["address"]; ok { var addr string if err := json.Unmarshal(addrRaw, &addr); err != nil || addr == "" { continue } src := Source{ID: pendingID, Address: addr} if d, ok := obj["district_name"]; ok { _ = json.Unmarshal(d, &src.District) } sources = append(sources, src) pendingID = "" } } if err := sc.Err(); err != nil { return nil, err } return sources, nil } // Build parses every source row into a canonical Entry and reports statistics. // Rows without a house number are still indexed for street lookups but counted // as unparsed, since a registry row without a house is suspicious. func Build(sources []Source) ([]address.Entry, Stats) { entries := make([]address.Entry, 0, len(sources)) streets := make(map[string]struct{}) seen := make(map[string]struct{}) stats := Stats{Total: len(sources)} for _, src := range sources { p := address.Parse(src.Address) if p.NameKey == "" { stats.Unparsed++ appendSample(&stats, src.Address) continue } entry := address.Entry{ ID: src.ID, Address: src.Address, District: src.District, Ordinal: p.Ordinal, NameKey: p.NameKey, NameDisplay: p.NameDisplay, StreetType: p.StreetType, StreetKey: p.StreetKey, House: p.House, HouseFrac: p.HouseFrac, HouseLetter: p.HouseLetter, Building: p.Building, BuildingFrac: p.BuildingFrac, BuildingLetter: p.BuildingLetter, Structure: p.Structure, } // The source registry contains rows that differ only by "стр." // (structure). Since structure does not affect which house an outage // belongs to, drop such duplicates rather than surfacing them as // spurious ambiguity at query time. if key := dedupKey(entry); key != "" { if _, ok := seen[key]; ok { stats.Duplicates++ continue } seen[key] = struct{}{} } entries = append(entries, entry) streets[p.StreetKey] = struct{}{} if p.HasHouse { stats.Parsed++ } else { stats.Unparsed++ appendSample(&stats, src.Address) } } stats.DistinctStreets = len(streets) sort.Strings(stats.UnparsedSample) return entries, stats } // dedupKey identifies the same house unit within the same street, ignoring the // structure component. func dedupKey(e address.Entry) string { if e.House == "" { return "" } return strings.Join([]string{ e.StreetKey, e.House, e.HouseFrac, e.HouseLetter, e.Building, e.BuildingFrac, e.BuildingLetter, }, "|") } func appendSample(s *Stats, address string) { const maxSamples = 20 if len(s.UnparsedSample) < maxSamples { s.UnparsedSample = append(s.UnparsedSample, address) } }