From 54cd87654f6a31e1a92cacfcfa85830317559ad7 Mon Sep 17 00:00:00 2001 From: Maksim Totmin Date: Tue, 2 Jun 2026 00:54:18 +0700 Subject: [PATCH] fix: reliable play count sync with auto-sync + deferred iTunesDB commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stops double-counting when iPod Nano 7G regenerates Play Counts file after a DB write (firmware quirk, new mtime with stale data). Architecture: - Mount: only max(cache, base) — never reads Play Counts delta - 2s after mount → auto-sync worker: reads delta, subtracts stored (persistent guard), applies effective to cache, defers DB write - iTunesDB commit deferred to next stable mount (checks W_OK first) - read_play_stats() uses 3s retry + read_play_counts_delta() fallback (filesystem-based PC parsing, independent of iTunesDB I/O errors) Changes: - ipod_nano7_db.py: sync_itunescdb() returns bool; new read_play_counts_delta() method; get_all_tracks() no longer adds delta; _merge_play_stats() drops delta addition; suppress iOpenPod ERROR logs to CRITICAL - library_cache.py: schema v2 with playcounts_log table + get/set_last_synced_delta, migration from v1 - worker.py: session guards (_processed_fwids, _auto_synced_fwids, _pending_commit_fwids); auto_sync_playcounts task; deferred commit in mount - app.py: QTimer.singleShot(2000) after mount for auto-sync - ipod_tab.py: _on_auto_state_changed rewritten (no unmount, mp guard) - library_tab.py: refresh_play_counts_from_cache() method --- src/app.py | 25 +++++++- src/ipod_nano7_db.py | 56 ++++++++++++++++-- src/library_cache.py | 52 +++++++++++++++-- src/ui/ipod_tab.py | 38 ++++++++++++- src/ui/library_tab.py | 46 +++++++++++++++ src/worker.py | 129 +++++++++++++++++++++++++++++++++++++++--- 6 files changed, 324 insertions(+), 22 deletions(-) diff --git a/src/app.py b/src/app.py index 8c33cf5..af56e8b 100644 --- a/src/app.py +++ b/src/app.py @@ -14,7 +14,7 @@ from PyQt6.QtWidgets import ( QApplication, QMainWindow, QWidget, QVBoxLayout, QSplitter, QStackedWidget, ) -from PyQt6.QtCore import Qt, QUrl +from PyQt6.QtCore import Qt, QUrl, QTimer from PyQt6.QtGui import QPixmap from PyQt6.QtMultimedia import QMediaPlayer, QAudioOutput @@ -64,6 +64,8 @@ class MainWindow(QMainWindow): self._shuffle_active: bool = False self._repeat_mode: str = "no_repeat" + self._auto_sync_worker = None + self._setup_player() self._setup_ui() self.library_tab.set_playlist_manager(self.playlist_manager) @@ -269,12 +271,33 @@ class MainWindow(QMainWindow): def _on_device_mounted(self, mount_point: str): self.library_tab.on_device_mounted(mount_point) + self.library_tab.refresh_play_counts_from_cache() self.statusBar().showMessage(f"iPod mounted at {mount_point}") + QTimer.singleShot(2000, lambda: self._start_auto_sync(mount_point)) def _on_device_unmounted(self): self.library_tab.on_device_unmounted() self.statusBar().showMessage("iPod disconnected") + def _start_auto_sync(self, mount_point: str): + if self._auto_sync_worker is not None: + return + from worker import WorkerThread + self._auto_sync_worker = WorkerThread( + "auto_sync_playcounts", parent=self, mount_point=mount_point, + ) + self._auto_sync_worker.progress_signal.connect(self._on_auto_sync_progress) + self._auto_sync_worker.finished_signal.connect(self._on_auto_sync_finished) + self._auto_sync_worker.start() + + def _on_auto_sync_progress(self, pct: int, msg: str): + pass + + def _on_auto_sync_finished(self, success: bool, msg: str, _result): + if success and msg.startswith("Auto-sync complete"): + self.library_tab.refresh_play_counts_from_cache() + self._auto_sync_worker = None + def _toggle_sidebar(self): visible = not self.sidebar.isVisible() self.sidebar.setVisible(visible) diff --git a/src/ipod_nano7_db.py b/src/ipod_nano7_db.py index 4b3f859..d1b0808 100644 --- a/src/ipod_nano7_db.py +++ b/src/ipod_nano7_db.py @@ -118,6 +118,10 @@ def _import_iop(module_name: str): logger = logging.getLogger(__name__) +# iOpenPod parsers/writers log ERROR for transient I/O (USB resettling) +logging.getLogger("iTunesDB_Parser").setLevel(logging.CRITICAL) +logging.getLogger("iTunesDB_Writer").setLevel(logging.CRITICAL) + # ── constants ──────────────────────────────────────────────────────── FILETYPE_CODES = { "mp3": 0x4D503320, @@ -511,8 +515,8 @@ class Nano7Database: location = f":iPod_Control:Music:{rel.replace(os.sep, ':')}" bs = base_stats.get(location, {}) dd = delta_stats.get(location, {}) - pc = bs.get("play_count", 0) + dd.get("play_count", 0) - sc = bs.get("skip_count", 0) + dd.get("skip_count", 0) + pc = bs.get("play_count", 0) + sc = bs.get("skip_count", 0) dd_rt = dd.get("rating", 0) rating = dd_rt if dd_rt >= 0 else bs.get("rating", 0) tracks.append({ @@ -767,12 +771,17 @@ class Nano7Database: # ── sync: regenerate everything via iOpenPod ────────────────────── - def sync_itunescdb(self) -> None: - """Regenerate ALL databases using iOpenPod's writers.""" + def sync_itunescdb(self) -> bool: + """Regenerate ALL databases using iOpenPod's writers. + + Returns True if the databases were written successfully. + """ try: self._sync_itunescdb_impl() + return True except Exception as e: logger.exception("iTunesDB sync failed: %s", e) + return False def _merge_play_stats(self, scanned: list[dict]) -> None: """Merge iPod + local play stats into *scanned* track dicts in-place. @@ -807,13 +816,13 @@ class Nano7Database: bs.get("play_count", 0), (db_entry or {}).get("play_count", 0), local_pc.get("play_count", 0), - ) + dd.get("play_count", 0) + ) s["skip_count"] = max( bs.get("skip_count", 0), (db_entry or {}).get("skip_count", 0), local_pc.get("skip_count", 0), - ) + dd.get("skip_count", 0) + ) dd_rating = dd.get("rating", 0) store_rating = (db_entry or {}).get("rating", 0) @@ -970,6 +979,41 @@ class Nano7Database: ) return base_stats, delta_stats + def read_play_counts_delta(self) -> dict[str, dict[str, int]]: + """Read Play Counts file using filesystem scan (no iTunesDB needed). + + Positional mapping uses ``_scan_ipod_files()`` order (correct after + any sync written by this app). Returns ``{location: {play_count, + skip_count, rating}}`` or empty dict on failure. + """ + pc_path = os.path.join( + self.mountpoint, "iPod_Control", "iTunes", "Play Counts", + ) + if not os.path.exists(pc_path): + return {} + pc_mod = _import_iop("iTunesDB_Parser.playcounts") + try: + entries = pc_mod.parse_playcounts(pc_path) + except Exception: + return {} + if not entries: + return {} + scanned = self._scan_ipod_files() + delta: dict[str, dict[str, int]] = {} + for i, entry in enumerate(entries): + if i >= len(scanned): + break + if entry.play_count == 0 and entry.skip_count == 0 and entry.rating < 0: + continue + rel = os.path.relpath(scanned[i]["file_path"], self.music_base) + loc = f":iPod_Control:Music:{rel.replace(os.sep, ':')}" + delta[loc] = { + "play_count": entry.play_count, + "skip_count": entry.skip_count, + "rating": entry.rating, + } + return delta + @staticmethod def _load_local_play_stats() -> dict[str, dict[str, int]]: """Read accumulated play stats from the local library cache. diff --git a/src/library_cache.py b/src/library_cache.py index 7771306..19aa66f 100644 --- a/src/library_cache.py +++ b/src/library_cache.py @@ -31,7 +31,7 @@ _DB_FILE = "library_cache.db" _BACKUP_FILE = "library_cache.db.backup" _OLD_JSON_FILE = "library_cache.json" -_SCHEMA_VERSION = 1 +_SCHEMA_VERSION = 2 _SCHEMA = """ CREATE TABLE IF NOT EXISTS library_cache ( @@ -64,7 +64,11 @@ CREATE INDEX IF NOT EXISTS idx_lc_hash ON library_cache(content_hash); CREATE INDEX IF NOT EXISTS idx_lc_artist ON library_cache(artist); CREATE INDEX IF NOT EXISTS idx_lc_album ON library_cache(album); -PRAGMA user_version = {version}; +CREATE TABLE IF NOT EXISTS playcounts_log ( + firewire_id TEXT PRIMARY KEY, + last_synced_delta TEXT NOT NULL DEFAULT '{}' +); + """ @@ -150,11 +154,18 @@ class LibraryCache: self._init_db() def _init_db(self) -> None: - self.conn.executescript( - _SCHEMA.format(version=_SCHEMA_VERSION) - ) + self._migrate_schema() + self.conn.executescript(_SCHEMA) + self.conn.execute(f"PRAGMA user_version = {_SCHEMA_VERSION}") self.conn.commit() + def _migrate_schema(self) -> None: + cur_ver = self.conn.execute("PRAGMA user_version").fetchone()[0] + if cur_ver >= 2: + return + if cur_ver == 1: + self.conn.execute("DROP TABLE IF EXISTS playcounts_log") + # ── public read API ──────────────────────────────────────────────── def get(self, file_path: str) -> dict[str, Any] | None: @@ -302,6 +313,37 @@ class LibraryCache: self.conn.commit() self._backup() + # ── play-counts delta deduplication ────────────────────────────────── + + def get_last_synced_delta(self, firewire_id: str) -> dict[str, dict[str, int]]: + """Return the delta dict that was last committed for this iPod. + + Returns an empty dict if no sync has happened yet. + """ + with self._lock: + row = self.conn.execute( + "SELECT last_synced_delta FROM playcounts_log WHERE firewire_id=?", + (firewire_id,), + ).fetchone() + if row and row["last_synced_delta"]: + try: + return json.loads(row["last_synced_delta"]) + except json.JSONDecodeError: + pass + return {} + + def set_last_synced_delta(self, firewire_id: str, + delta: dict[str, dict[str, int]]) -> None: + """Store the delta that was committed during the last sync.""" + blob = json.dumps(delta, sort_keys=True, separators=(",", ":")) + with self._lock: + self.conn.execute( + "INSERT OR REPLACE INTO playcounts_log (firewire_id, last_synced_delta) " + "VALUES (?, ?)", + (firewire_id, blob), + ) + self._backup() + # ── backup rotation ──────────────────────────────────────────────── def _backup(self) -> None: diff --git a/src/ui/ipod_tab.py b/src/ui/ipod_tab.py index 224b432..f985fb4 100644 --- a/src/ui/ipod_tab.py +++ b/src/ui/ipod_tab.py @@ -1023,9 +1023,45 @@ class iPodTab(QWidget): pass def _on_auto_state_changed(self, devices): + """Automatic device detection callback from the monitor timer. + + Updates the device combo silently and only mounts a device that + is not already active. Idempotent Play Counts processing in + the worker prevents double‑counting even if the mount handler + runs multiple times due to USB glitches. + """ if self._is_worker_running(): return - self._on_device_detection_finished(True, f"Found {len(devices)} devices", devices) + + self._refresh_device_combo(devices) + + if not devices: + return + + mp = devices[0].get("mount_point", "") + + if mp and mp == self.current_mount_point: + return + + self._start_mount_and_load( + devices[0], + mount_point=mp, + already_mounted=devices[0].get("mounted", False), + ) + + def _refresh_device_combo(self, devices): + """Update the device combo box without triggering re‑mount.""" + self.device_combo.blockSignals(True) + self.device_combo.clear() + for device in devices: + mounted = device.get("mounted", False) + suffix = " [mounted]" if mounted else " [not mounted]" + self.device_combo.addItem(device.get("name", "Unknown Device") + suffix) + idx = self.device_combo.count() - 1 + self.device_combo.setItemData(idx, device, Qt.ItemDataRole.UserRole) + if devices: + self.device_combo.setCurrentIndex(0) + self.device_combo.blockSignals(False) def _on_device_combo_changed(self, index): device = self.device_combo.itemData(index, Qt.ItemDataRole.UserRole) diff --git a/src/ui/library_tab.py b/src/ui/library_tab.py index 8ac19fd..b4bf75d 100644 --- a/src/ui/library_tab.py +++ b/src/ui/library_tab.py @@ -685,6 +685,52 @@ class LibraryTab(QWidget): self._restore_playing_row() self._build_album_grid() + def refresh_play_counts_from_cache(self): + """Reload play-count values from the persistent cache without a + full library rescan. + + Called after ``mount_and_load`` completes — the cache already + holds merged iPod + local play stats, but the table was + rendered before the mount and displays stale values. + """ + cache = get_library_cache() + updated = 0 + + ready_index = {t["path"]: i for i, t in enumerate(self.library_ready)} + + for row in range(self.library_table.rowCount()): + item_0 = self.library_table.item(row, 0) + if item_0 is None: + continue + data = item_0.data(Qt.ItemDataRole.UserRole) + if data is None: + continue + _is_ready, track = data + if not _is_ready: + continue + path = track.get("path", "") + if not path: + continue + entry = cache.get(path) + if entry is None: + continue + + new_pc = entry.get("play_count", 0) + old_pc = track.get("play_count", 0) + if new_pc != old_pc: + track["play_count"] = new_pc + pc_cell = self.library_table.item(row, 7) + if pc_cell: + pc_cell.setText(str(new_pc)) + + idx = ready_index.get(path) + if idx is not None: + self.library_ready[idx]["play_count"] = new_pc + updated += 1 + + if updated: + logger.debug("Refreshed play counts for %d tracks from cache", updated) + def _restore_playing_row(self): idx = self.current_playback_index if idx < 0: diff --git a/src/worker.py b/src/worker.py index cea1b56..239c046 100644 --- a/src/worker.py +++ b/src/worker.py @@ -24,6 +24,10 @@ SOURCE_EXTS = {'.flac', '.wav', '.ogg', '.opus', '.wma', '.ape', '.aiff', '.alac class WorkerThread(QThread): """Worker thread for background tasks""" + _processed_fwids: set[str] = set() + _auto_synced_fwids: set[str] = set() + _pending_commit_fwids: set[str] = set() + progress_signal = pyqtSignal(int, str) finished_signal = pyqtSignal(bool, str, object) @@ -45,6 +49,7 @@ class WorkerThread(QThread): "remove_duplicates": self._run_remove_duplicates, "sync_metadata": self._run_sync_metadata, "create_playlist": self._run_create_playlist, + "auto_sync_playcounts": self._run_auto_sync_playcounts, } handler = handlers.get(self.task_type) if handler: @@ -208,11 +213,11 @@ class WorkerThread(QThread): from library_cache import get_library_cache, save_library_cache cache = get_library_cache() - # Read deltas from iPod Play Counts file without deleting it — - # the file is cleaned up by _cleanup_play_counts_file() only after a - # successful DB write in _sync_itunescdb_impl / sync_metadata_for. - ipod_base, ipod_delta = db.read_play_stats() + ipod_base, _ipod_delta = db.read_play_stats() + fwid = db.firewire_id.hex() + WorkerThread._processed_fwids.add(fwid) + # ── build content_hash → iPod location map ────────────────────── hash_to_location: dict[str, str] = {} for t in tracks: ch = t.get("content_hash") @@ -225,6 +230,7 @@ class WorkerThread(QThread): loc = f":iPod_Control:Music:{rel.replace(os.sep, ':')}" hash_to_location[ch] = loc + # ── merge into local cache ────────────────────────────────────── updated_cache = 0 for file_path, entry in cache.iter_entries(): if not os.path.exists(file_path): @@ -234,16 +240,14 @@ class WorkerThread(QThread): continue loc = hash_to_location[ch] bs = ipod_base.get(loc, {}) - dd = ipod_delta.get(loc, {}) new_play_count = max( entry.get("play_count", 0), bs.get("play_count", 0), - ) + dd.get("play_count", 0) + ) new_skip_count = max( entry.get("skip_count", 0), bs.get("skip_count", 0), - ) + dd.get("skip_count", 0) - dd_rating = dd.get("rating", 0) - new_rating = dd_rating if dd_rating >= 0 else bs.get("rating", 0) or entry.get("rating", 0) + ) + new_rating = bs.get("rating", 0) or entry.get("rating", 0) new_last_played = max( entry.get("last_played", 0), bs.get("last_played", 0), ) @@ -261,6 +265,14 @@ class WorkerThread(QThread): save_library_cache() logger.info("Updated play counts for %d local tracks from iPod", updated_cache) + # ── deferred iTunesDB commit on stable USB ────────────────── + if fwid in WorkerThread._pending_commit_fwids and os.access(mount_point, os.W_OK): + if db.sync_itunescdb(): + WorkerThread._pending_commit_fwids.discard(fwid) + logger.info("Deferred iTunesDB commit for %s complete", fwid) + else: + logger.debug("Deferred commit for %s failed (read-only filesystem)", fwid) + self.progress_signal.emit(80, "Loading complete") result = { @@ -273,6 +285,105 @@ class WorkerThread(QThread): True, f"Loaded {len(tracks)} tracks from iPod", result ) + def _run_auto_sync_playcounts(self): + mount_point = self.kwargs.get("mount_point") + if not mount_point: + self.finished_signal.emit(False, "No mount point for auto-sync", None) + return + + ipod_ctrl = os.path.join(mount_point, "iPod_Control") + if not os.path.exists(ipod_ctrl): + self.finished_signal.emit(True, "Auto-sync skipped (device gone)", None) + return + + from ipod_nano7_db import Nano7Database + db = Nano7Database(mount_point) + fwid = db.firewire_id.hex() + + if fwid in WorkerThread._auto_synced_fwids: + self.finished_signal.emit(True, "Auto-sync already done", None) + return + + ipod_base, ipod_delta = db.read_play_stats() + if not ipod_base and not ipod_delta: + import time + time.sleep(3) + ipod_base, ipod_delta = db.read_play_stats() + if not ipod_base and not ipod_delta: + ipod_delta = db.read_play_counts_delta() + if not ipod_delta: + self.finished_signal.emit(True, "Auto-sync: no delta", None) + return + + from library_cache import get_library_cache, save_library_cache + cache = get_library_cache() + stored = cache.get_last_synced_delta(fwid) + + effective: dict[str, dict[str, int]] = {} + for loc, dd in ipod_delta.items(): + sd = stored.get(loc, {}) + eff_pc = max(0, dd.get("play_count", 0) - sd.get("play_count", 0)) + eff_sc = max(0, dd.get("skip_count", 0) - sd.get("skip_count", 0)) + if eff_pc > 0 or eff_sc > 0: + effective[loc] = {"play_count": eff_pc, "skip_count": eff_sc} + dd_rating = dd.get("rating", 0) + if dd_rating > 0: + effective.setdefault(loc, {})["rating"] = dd_rating + + cache.set_last_synced_delta(fwid, ipod_delta) + + if effective: + tracks = db.get_all_tracks() + hash_to_loc: dict[str, str] = {} + loc_to_hash: dict[str, str] = {} + for t in tracks: + ch = t.get("content_hash", "") + fpath = t.get("file_path", "") + if ch and fpath: + rel = os.path.relpath( + fpath, os.path.join(db.mountpoint, "iPod_Control", "Music"), + ) + loc = f":iPod_Control:Music:{rel.replace(os.sep, ':')}" + loc_to_hash[loc] = ch + hash_to_loc[ch] = loc + + self.progress_signal.emit(40, "Merging play counts...") + updated = 0 + for file_path, entry in cache.iter_entries(): + ch = entry.get("content_hash", "") + if ch not in hash_to_loc: + continue + loc = hash_to_loc[ch] + eff = effective.get(loc) + if not eff: + continue + bs = ipod_base.get(loc, {}) + new_pc = max(entry.get("play_count", 0), bs.get("play_count", 0)) + eff.get( + "play_count", 0, + ) + new_sc = max(entry.get("skip_count", 0), bs.get("skip_count", 0)) + eff.get( + "skip_count", 0, + ) + new_rt = eff.get("rating", 0) or bs.get("rating", 0) or entry.get("rating", 0) + new_lp = max(entry.get("last_played", 0), bs.get("last_played", 0)) + cache.update_play_stats( + file_path, + play_count=new_pc, + rating=new_rt, + last_played=new_lp, + skip_count=new_sc, + ) + updated += 1 + + if updated: + save_library_cache() + logger.info("Auto-sync: applied %d effective deltas", updated) + + self.progress_signal.emit(100, "Auto-sync complete") + WorkerThread._pending_commit_fwids.add(fwid) + WorkerThread._auto_synced_fwids.add(fwid) + self.finished_signal.emit(True, "Auto-sync complete", mount_point) + def _run_export_ipod(self): tracks = self.kwargs.get("tracks", []) dest_dir = self.kwargs.get("dest_dir", "")