From 59cd98afbf8ef65fb01fc81dca3eb7092c60f164 Mon Sep 17 00:00:00 2001 From: Christian Heller Date: Sat, 4 Jan 2025 18:47:42 +0100 Subject: [PATCH] Re-organize command calling of sub-scripts. --- src/migrate.py | 8 ----- src/run.py | 31 ++++++++++++++++++ src/serve.py | 16 --------- src/ytplom/db.py | 12 ++++--- src/ytplom/http.py | 17 ++++++++-- src/ytplom/migrations.py | 12 +++++-- src/{ => ytplom}/sync.py | 70 +++++++++++++++++++--------------------- ytplom | 7 +--- 8 files changed, 97 insertions(+), 76 deletions(-) delete mode 100755 src/migrate.py create mode 100755 src/run.py delete mode 100755 src/serve.py rename src/{ => ytplom}/sync.py (75%) mode change 100755 => 100644 diff --git a/src/migrate.py b/src/migrate.py deleted file mode 100755 index 9d0dcc6..0000000 --- a/src/migrate.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 -"""Script to migrate DB to most recent schema.""" -from ytplom.db import DbFile -from ytplom.migrations import MIGRATIONS - - -if __name__ == '__main__': - DbFile(expected_version=-1).migrate(MIGRATIONS) diff --git a/src/run.py b/src/run.py new file mode 100755 index 0000000..4b1ddc3 --- /dev/null +++ b/src/run.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Match command line calls to appropriate scripts.""" + +# included libs +from sys import argv, exit as sys_exit +# ourselves +from ytplom.db import DbFile +from ytplom.primitives import HandledException +from ytplom.migrations import migrate +from ytplom.http import serve +from ytplom.sync import sync + + +if __name__ == '__main__': + try: + if 2 != len(argv): + raise HandledException('Bad number of command arguments.') + match argv[1]: + case 'create_db': + DbFile.create() + case 'migrate_db': + migrate() + case 'serve': + serve() + case 'sync': + sync() + case _: + raise HandledException('Unknown argument.') + except HandledException as e: + print(e) + sys_exit(1) diff --git a/src/serve.py b/src/serve.py deleted file mode 100755 index ef86560..0000000 --- a/src/serve.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -"""Minimalistic download-focused YouTube interface.""" -from ytplom.misc import Config -from ytplom.http import Server - - -if __name__ == '__main__': - config = Config() - server = Server(config) - print(f'running at port {config.port}') - try: - server.serve_forever() - except KeyboardInterrupt: - print('aborted due to keyboard interrupt; ' - 'repeat to end download thread too') - server.server_close() diff --git a/src/ytplom/db.py b/src/ytplom/db.py index f503e9b..e30e2ad 100644 --- a/src/ytplom/db.py +++ b/src/ytplom/db.py @@ -1,15 +1,19 @@ """Database access and management code.""" + +# included libs from base64 import urlsafe_b64decode, urlsafe_b64encode from hashlib import file_digest from pathlib import Path from sqlite3 import (connect as sql_connect, Connection as SqlConnection, Cursor as SqlCursor, Row as SqlRow) from typing import Callable, Literal, NewType, Optional, Self +# ourselves from ytplom.primitives import ( HandledException, NotFoundException, PATH_APP_DATA) + EXPECTED_DB_VERSION = 6 -PATH_DB = PATH_APP_DATA.joinpath('TESTdb.sql') +PATH_DB = PATH_APP_DATA.joinpath('db.sql') SqlText = NewType('SqlText', str) MigrationsList = list[tuple[Path, Optional[Callable]]] @@ -63,14 +67,12 @@ class DbFile: ) -> None: self._path = path if not path.is_file(): - raise HandledException( - f'no DB file at {path} – run "create"?') + raise HandledException(f'no DB file at {path}') if expected_version >= 0: user_version = self._get_user_version() if user_version != expected_version: raise HandledException( - f'wrong database version {user_version}, expected: ' - f'{expected_version} – run "migrate"?') + f'wrong DB version {user_version} (!= {expected_version})') def _get_user_version(self) -> int: with sql_connect(self._path) as conn: diff --git a/src/ytplom/http.py b/src/ytplom/http.py index ad2cc85..5fef79d 100644 --- a/src/ytplom/http.py +++ b/src/ytplom/http.py @@ -64,8 +64,8 @@ class Server(ThreadingHTTPServer): """Extension of parent server providing for Player and DownloadsManager.""" def __init__(self, config: Config, *args, **kwargs) -> None: - super().__init__((config.host, config.port), _TaskHandler, - *args, **kwargs) + super().__init__( + (config.host, config.port), _TaskHandler, *args, **kwargs) self.config = config self.jinja = JinjaEnv(loader=JinjaFSLoader(_PATH_TEMPLATES)) self.player = Player(config.whitelist_tags_display, @@ -416,3 +416,16 @@ class _TaskHandler(BaseHTTPRequestHandler): {'selected': 'playlist', 'filter_path': self.server.player.filter_path, 'needed_tags': self.server.player.needed_tags.joined}) + + +def serve(): + """Do Server.serve_forever on .config.port until keyboard interrupt.""" + config = Config() + server = Server(Config()) + print(f'running at port {config.port}') + try: + server.serve_forever() + except KeyboardInterrupt: + print('aborted due to keyboard interrupt; ' + 'repeat to end download thread too') + server.server_close() diff --git a/src/ytplom/migrations.py b/src/ytplom/migrations.py index 59c21f3..cb038ef 100644 --- a/src/ytplom/migrations.py +++ b/src/ytplom/migrations.py @@ -1,7 +1,10 @@ """Anything pertaining specifically to DB migrations.""" + +# included libs from pathlib import Path from typing import Callable -from ytplom.db import DbConn, MigrationsList, SqlText +# ourselves +from ytplom.db import DbConn, DbFile, MigrationsList, SqlText from ytplom.primitives import HandledException @@ -52,7 +55,7 @@ def _mig_4_convert_digests(conn: DbConn) -> None: _rewrite_files_last_field_processing_first_field(conn, bytes.fromhex) -MIGRATIONS: MigrationsList = [ +_MIGRATIONS: MigrationsList = [ (Path('0_init.sql'), None), (Path('1_add_files_last_updated.sql'), None), (Path('2_add_files_sha512.sql'), _mig_2_calc_digests), @@ -61,3 +64,8 @@ MIGRATIONS: MigrationsList = [ (Path('5_files_redo.sql'), None), (Path('6_add_files_tags.sql'), None) ] + + +def migrate(): + """Migrate DB file at expected default path to most recent version.""" + DbFile(expected_version=-1).migrate(_MIGRATIONS) diff --git a/src/sync.py b/src/ytplom/sync.py old mode 100755 new mode 100644 similarity index 75% rename from src/sync.py rename to src/ytplom/sync.py index a6be397..89a198d --- a/src/sync.py +++ b/src/ytplom/sync.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 -"""Script to sync between local and remote instances.""" +"""To sync between local and remote instances.""" # included libs from typing import Any, Callable @@ -8,30 +7,31 @@ from urllib.request import Request, urlopen # non-included libs from paramiko import SSHClient # type: ignore from scp import SCPClient # type: ignore +# ourselves from ytplom.db import DbConn, DbFile, Hash, PATH_DB from ytplom.misc import (PATH_TEMP, Config, FlagName, QuotaLog, VideoFile, YoutubeQuery, YoutubeVideo) from ytplom.http import PAGE_NAMES -PATH_DB_REMOTE = PATH_TEMP.joinpath('remote_db.sql') -ATTR_NAME_LAST_UPDATE = 'last_update' +_PATH_DB_REMOTE = PATH_TEMP.joinpath('remote_db.sql') +_ATTR_NAME_LAST_UPDATE = 'last_update' -def back_and_forth(sync_func: Callable, - dbs: tuple[DbConn, DbConn], - shared: Any - ) -> None: +def _back_and_forth(sync_func: Callable, + dbs: tuple[DbConn, DbConn], + shared: Any + ) -> None: """Run sync_func on arg pairs + shared, and again with pairs switched.""" host_names = 'local', 'remote' sync_func(host_names, dbs, shared) sync_func(*(tuple(reversed(list(t))) for t in (host_names, dbs)), shared) -def sync_objects(host_names: tuple[str, str], - dbs: tuple[DbConn, DbConn], - cls: Any - ) -> None: +def _sync_objects(host_names: tuple[str, str], + dbs: tuple[DbConn, DbConn], + cls: Any + ) -> None: """Equalize both DB's object collections; prefer newer states to older.""" id_name = 'id_' if 'id' == cls.id_name else cls.id_name obj_colls = cls.get_all(dbs[0]), cls.get_all(dbs[1]) @@ -45,9 +45,9 @@ def sync_objects(host_names: tuple[str, str], for obj_1 in [obj for obj in obj_colls[1] # pick those from 2nd coll if id_ == getattr(obj, id_name)]: # of same ID as obj_0 msg_verb = 'updating' - if hasattr(obj_0, ATTR_NAME_LAST_UPDATE): - last_update_0 = getattr(obj_0, ATTR_NAME_LAST_UPDATE) - last_update_1 = getattr(obj_1, ATTR_NAME_LAST_UPDATE) + if hasattr(obj_0, _ATTR_NAME_LAST_UPDATE): + last_update_0 = getattr(obj_0, _ATTR_NAME_LAST_UPDATE) + last_update_1 = getattr(obj_1, _ATTR_NAME_LAST_UPDATE) sync_down = last_update_0 > last_update_1 if not sync_down: direction = f'{host_names[0]}->{host_names[1]}' @@ -58,10 +58,10 @@ def sync_objects(host_names: tuple[str, str], print(f'SYNC {cls.__name__}: {msg_verb} {direction} {id_} {to_str}') -def sync_relations(host_names: tuple[str, str], - dbs: tuple[DbConn, DbConn], - yt_result: YoutubeVideo - ) -> None: +def _sync_relations(host_names: tuple[str, str], + dbs: tuple[DbConn, DbConn], + yt_result: YoutubeVideo + ) -> None: """To dbs[1] add YT yt_video->yt_q_colls[0] mapping not in yt_q_colls[1]""" yt_q_colls = tuple(YoutubeQuery.get_all_for_video(db, yt_result.id_) for db in dbs) @@ -72,20 +72,20 @@ def sync_relations(host_names: tuple[str, str], yt_result.save_to_query(dbs[1], q.id_) -def sync_dbs(scp: SCPClient) -> None: +def _sync_dbs(scp: SCPClient) -> None: """Download remote DB, run sync_(objects|relations), put remote DB back.""" - scp.get(PATH_DB, PATH_DB_REMOTE) + scp.get(PATH_DB, _PATH_DB_REMOTE) with DbConn(DbFile(PATH_DB).connect()) as db_local, \ - DbConn(DbFile(PATH_DB_REMOTE).connect()) as db_remote: + DbConn(DbFile(_PATH_DB_REMOTE).connect()) as db_remote: for cls in (QuotaLog, YoutubeQuery, YoutubeVideo, VideoFile): - back_and_forth(sync_objects, (db_local, db_remote), cls) + _back_and_forth(_sync_objects, (db_local, db_remote), cls) for yt_video_local in YoutubeVideo.get_all(db_local): - back_and_forth(sync_relations, (db_local, db_remote), - yt_video_local) + _back_and_forth(_sync_relations, (db_local, db_remote), + yt_video_local) for db in (db_remote, db_local): db.commit() - scp.put(PATH_DB_REMOTE, PATH_DB) - PATH_DB_REMOTE.unlink() + scp.put(_PATH_DB_REMOTE, PATH_DB) + _PATH_DB_REMOTE.unlink() def _urls_here_and_there(config: Config, page_name: str) -> tuple[str, ...]: @@ -94,14 +94,14 @@ def _urls_here_and_there(config: Config, page_name: str) -> tuple[str, ...]: (config.host, config.port))) -def purge_deleteds(config: Config) -> None: +def _purge_deleteds(config: Config) -> None: """Command both servers to actually purge "deleted" files.""" for url_purge in _urls_here_and_there(config, 'purge'): with urlopen(Request(url_purge, method='POST')) as response: print(f'SYNC: Calling purge via {url_purge} – {response.read()}') -def fill_missing(scp: SCPClient, config: Config) -> None: +def _fill_missing(scp: SCPClient, config: Config) -> None: """Between config.host and .remote, fill files listed in as missing.""" missings = [] for url_missing in _urls_here_and_there(config, 'missing'): @@ -122,17 +122,13 @@ def fill_missing(scp: SCPClient, config: Config) -> None: mover(vf.full_path, vf.full_path) -def main(): +def sync(): """Connect to remote, sync local+remote DBs, + downloads where missing.""" config = Config() ssh = SSHClient() ssh.load_system_host_keys() ssh.connect(config.remote) with SCPClient(ssh.get_transport()) as scp: - sync_dbs(scp) - purge_deleteds(config) - fill_missing(scp, config) - - -if __name__ == '__main__': - main() + _sync_dbs(scp) + _purge_deleteds(config) + _fill_missing(scp, config) diff --git a/ytplom b/ytplom index 7e5c4e2..9bea676 100755 --- a/ytplom +++ b/ytplom @@ -4,14 +4,9 @@ set -e PATH_APP_SHARE=~/.local/share/ytplom PATH_VENV="${PATH_APP_SHARE}/venv" -if [ ! "$1" = 'serve' ] && [ ! "$1" = 'sync' ] && [ ! "$1" = 'migrate' ] && [ ! "$1" = 'create' ]; then - echo "Need argument ('serve' or 'sync' or 'migrate' or 'create')." - false -fi - python3 -m venv "${PATH_VENV}" . "${PATH_VENV}/bin/activate" echo "Checking dependencies." pip3 install -r "${PATH_APP_SHARE}/requirements.txt" export PYTHONPATH="${PATH_APP_SHARE}:${PYTHONPATH}" -python3 "${PATH_APP_SHARE}/${1}.py" +python3 "${PATH_APP_SHARE}/run.py" $@ -- 2.30.2