From: Christian Heller <c.heller@plomlompom.de>
Date: Thu, 26 Dec 2024 08:07:07 +0000 (+0100)
Subject: On sync, rather than only unlink "deleted" files, also remove their DB entries.
X-Git-Url: https://plomlompom.com/repos/%7B%7Bdb.prefix%7D%7D/static/%7B%7B%20web_path%20%7D%7D/%7B%7Bprefix%7D%7D/template?a=commitdiff_plain;h=007991cc45385ffc1b994bb88cdf389a16c3dff9;p=ytplom

On sync, rather than only unlink "deleted" files, also remove their DB entries.
---

diff --git a/src/sync.py b/src/sync.py
index c3ce570..f9fcb74 100755
--- a/src/sync.py
+++ b/src/sync.py
@@ -4,7 +4,7 @@
 # included libs
 from typing import Any, Callable
 from json import loads as json_loads
-from urllib.request import urlopen
+from urllib.request import Request, urlopen
 # non-included libs
 from paramiko import SSHClient  # type: ignore
 from scp import SCPClient  # type: ignore
@@ -81,18 +81,29 @@ def sync_dbs(scp: SCPClient) -> None:
         for yt_video_local in YoutubeVideo.get_all(db_local):
             back_and_forth(sync_relations, (db_local, db_remote),
                            yt_video_local)
-        db_remote.commit()
-        db_local.commit()
+        for db in (db_remote, db_local):
+            db.commit()
     scp.put(PATH_DB_REMOTE, PATH_DB)
     PATH_DB_REMOTE.unlink()
 
 
+def _urls_here_and_there(config: Config, page_name: str) -> tuple[str, ...]:
+    return tuple(f'http://{host}:{port}/{PAGE_NAMES[page_name]}'
+                 for host, port in ((config.remote, config.port_remote),
+                                    (config.host, config.port)))
+
+
+def purge_deleteds(config: Config) -> None:
+    """Command both servers to actually purge "deleted" files."""
+    for url_purge in _urls_here_and_there(config, 'purge'):
+        with urlopen(Request(url_purge, method='POST')) as response:
+            print(f'SYNC: Calling purge via {url_purge} – {response.read()}')
+
+
 def fill_missing(scp: SCPClient, config: Config) -> None:
     """Between config.host and .remote, fill files listed in as missing."""
     missings = []
-    for host, port in ((config.remote, config.port_remote),
-                       (config.host, config.port)):
-        url_missing = f'http://{host}:{port}/{PAGE_NAMES["missing"]}'
+    for url_missing in _urls_here_and_there(config, 'missing'):
         with urlopen(url_missing) as response:
             missings += [list(json_loads(response.read()))]
     conn = DbConn()
@@ -117,6 +128,7 @@ def main():
     ssh.connect(config.remote)
     with SCPClient(ssh.get_transport()) as scp:
         sync_dbs(scp)
+        purge_deleteds(config)
         fill_missing(scp, config)
 
 
diff --git a/src/ytplom/http.py b/src/ytplom/http.py
index 11b29c0..aa1b663 100644
--- a/src/ytplom/http.py
+++ b/src/ytplom/http.py
@@ -41,6 +41,7 @@ PAGE_NAMES: dict[str, Path] = {
     'missing': Path('missing'),
     'player': Path('player'),
     'playlist': Path('playlist'),
+    'purge': Path('purge'),
     'thumbnails': Path('thumbnails'),
     'yt_result': Path('yt_result'),
     'yt_query': Path('yt_query'),
@@ -141,6 +142,14 @@ class _TaskHandler(BaseHTTPRequestHandler):
             self._receive_yt_query(QueryText(postvars.first_for('query')))
         elif PAGE_NAMES['player'] == page_name:
             self._receive_player_command(postvars)
+        elif PAGE_NAMES['purge'] == page_name:
+            self._purge_deleted_files()
+
+    def _purge_deleted_files(self) -> None:
+        with DbConn() as conn:
+            VideoFile.purge_deleteds(conn)
+            conn.commit()
+        self._send_http('OK', code=200)
 
     def _receive_player_command(self, postvars: _ReqMap) -> None:
         command = postvars.first_for('command')
diff --git a/src/ytplom/misc.py b/src/ytplom/misc.py
index 7821b1f..7458cb7 100644
--- a/src/ytplom/misc.py
+++ b/src/ytplom/misc.py
@@ -480,14 +480,25 @@ class VideoFile(DbData):
     def ensure_absence_if_deleted(self) -> None:
         """If 'delete' flag set, ensure no actual file in filesystem."""
         if self.is_flag_set(FlagName('delete')) and self.present:
-            print(f'SYNC: {self.rel_path} set "delete", '
-                  'removing from filesystem.')
             self.unlink_locally()
 
     def unlink_locally(self) -> None:
         """Remove actual file from local filesystem."""
+        print(f'SYNC: removing from filesystem: {self.rel_path}')
         self.full_path.unlink()
 
+    @classmethod
+    def purge_deleteds(cls, conn: BaseDbConn) -> None:
+        """For all of .is_flag_set("deleted"), remove file _and_ DB entry."""
+        for file in [f for f in cls.get_all(conn)
+                     if f.is_flag_set(FlagName('delete'))]:
+            if file.present:
+                file.unlink_locally()
+            print(f'SYNC: purging off DB: {file.digest.b64} ({file.rel_path})')
+            conn.exec(
+                    SqlText(f'DELETE FROM {cls._table_name} WHERE digest = ?'),
+                    (file.digest.bytes,))
+
 
 class QuotaLog(DbData):
     """Collects API access quota costs."""