From: Christian Heller Date: Sat, 4 Jan 2025 17:47:42 +0000 (+0100) Subject: Re-organize command calling of sub-scripts. X-Git-Url: https://plomlompom.com/repos/%7B%7Bdb.prefix%7D%7D/static/%7B%7B%20web_path%20%7D%7D/blog?a=commitdiff_plain;h=59cd98afbf8ef65fb01fc81dca3eb7092c60f164;p=ytplom Re-organize command calling of sub-scripts. --- diff --git a/src/migrate.py b/src/migrate.py deleted file mode 100755 index 9d0dcc6..0000000 --- a/src/migrate.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 -"""Script to migrate DB to most recent schema.""" -from ytplom.db import DbFile -from ytplom.migrations import MIGRATIONS - - -if __name__ == '__main__': - DbFile(expected_version=-1).migrate(MIGRATIONS) diff --git a/src/run.py b/src/run.py new file mode 100755 index 0000000..4b1ddc3 --- /dev/null +++ b/src/run.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Match command line calls to appropriate scripts.""" + +# included libs +from sys import argv, exit as sys_exit +# ourselves +from ytplom.db import DbFile +from ytplom.primitives import HandledException +from ytplom.migrations import migrate +from ytplom.http import serve +from ytplom.sync import sync + + +if __name__ == '__main__': + try: + if 2 != len(argv): + raise HandledException('Bad number of command arguments.') + match argv[1]: + case 'create_db': + DbFile.create() + case 'migrate_db': + migrate() + case 'serve': + serve() + case 'sync': + sync() + case _: + raise HandledException('Unknown argument.') + except HandledException as e: + print(e) + sys_exit(1) diff --git a/src/serve.py b/src/serve.py deleted file mode 100755 index ef86560..0000000 --- a/src/serve.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -"""Minimalistic download-focused YouTube interface.""" -from ytplom.misc import Config -from ytplom.http import Server - - -if __name__ == '__main__': - config = Config() - server = Server(config) - print(f'running at port {config.port}') - try: - server.serve_forever() - except KeyboardInterrupt: - print('aborted due to keyboard interrupt; ' - 'repeat to end download thread too') - server.server_close() diff --git a/src/sync.py b/src/sync.py deleted file mode 100755 index a6be397..0000000 --- a/src/sync.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python3 -"""Script to sync between local and remote instances.""" - -# included libs -from typing import Any, Callable -from json import loads as json_loads -from urllib.request import Request, urlopen -# non-included libs -from paramiko import SSHClient # type: ignore -from scp import SCPClient # type: ignore -from ytplom.db import DbConn, DbFile, Hash, PATH_DB -from ytplom.misc import (PATH_TEMP, Config, FlagName, QuotaLog, VideoFile, - YoutubeQuery, YoutubeVideo) -from ytplom.http import PAGE_NAMES - - -PATH_DB_REMOTE = PATH_TEMP.joinpath('remote_db.sql') -ATTR_NAME_LAST_UPDATE = 'last_update' - - -def back_and_forth(sync_func: Callable, - dbs: tuple[DbConn, DbConn], - shared: Any - ) -> None: - """Run sync_func on arg pairs + shared, and again with pairs switched.""" - host_names = 'local', 'remote' - sync_func(host_names, dbs, shared) - sync_func(*(tuple(reversed(list(t))) for t in (host_names, dbs)), shared) - - -def sync_objects(host_names: tuple[str, str], - dbs: tuple[DbConn, DbConn], - cls: Any - ) -> None: - """Equalize both DB's object collections; prefer newer states to older.""" - id_name = 'id_' if 'id' == cls.id_name else cls.id_name - obj_colls = cls.get_all(dbs[0]), cls.get_all(dbs[1]) - for obj_0 in [obj for obj in obj_colls[0] # only pick objs without equal - if obj not in obj_colls[1]]: # in 2nd coll, even if same ID - id_ = getattr(obj_0, id_name) - sync_down = True - to_str = obj_0 - msg_verb = 'adding' - direction = f'{host_names[1]}->{host_names[0]}' - for obj_1 in [obj for obj in obj_colls[1] # pick those from 2nd coll - if id_ == getattr(obj, id_name)]: # of same ID as obj_0 - msg_verb = 'updating' - if hasattr(obj_0, ATTR_NAME_LAST_UPDATE): - last_update_0 = getattr(obj_0, ATTR_NAME_LAST_UPDATE) - last_update_1 = getattr(obj_1, ATTR_NAME_LAST_UPDATE) - sync_down = last_update_0 > last_update_1 - if not sync_down: - direction = f'{host_names[0]}->{host_names[1]}' - to_str = obj_1 - obj_1.save(dbs[0]) - if sync_down: - obj_0.save(dbs[1]) - print(f'SYNC {cls.__name__}: {msg_verb} {direction} {id_} {to_str}') - - -def sync_relations(host_names: tuple[str, str], - dbs: tuple[DbConn, DbConn], - yt_result: YoutubeVideo - ) -> None: - """To dbs[1] add YT yt_video->yt_q_colls[0] mapping not in yt_q_colls[1]""" - yt_q_colls = tuple(YoutubeQuery.get_all_for_video(db, yt_result.id_) - for db in dbs) - direction = f'adding {host_names[1]}->{host_names[0]} mapping' - result = f'result {yt_result.id_} ({yt_result})' - for q in [q for q in yt_q_colls[0] if q not in yt_q_colls[1]]: - print(f'SYNC: {direction} of query {q.id_} ({q}) to {result}') - yt_result.save_to_query(dbs[1], q.id_) - - -def sync_dbs(scp: SCPClient) -> None: - """Download remote DB, run sync_(objects|relations), put remote DB back.""" - scp.get(PATH_DB, PATH_DB_REMOTE) - with DbConn(DbFile(PATH_DB).connect()) as db_local, \ - DbConn(DbFile(PATH_DB_REMOTE).connect()) as db_remote: - for cls in (QuotaLog, YoutubeQuery, YoutubeVideo, VideoFile): - back_and_forth(sync_objects, (db_local, db_remote), cls) - for yt_video_local in YoutubeVideo.get_all(db_local): - back_and_forth(sync_relations, (db_local, db_remote), - yt_video_local) - for db in (db_remote, db_local): - db.commit() - scp.put(PATH_DB_REMOTE, PATH_DB) - PATH_DB_REMOTE.unlink() - - -def _urls_here_and_there(config: Config, page_name: str) -> tuple[str, ...]: - return tuple(f'http://{host}:{port}/{PAGE_NAMES[page_name]}' - for host, port in ((config.remote, config.port_remote), - (config.host, config.port))) - - -def purge_deleteds(config: Config) -> None: - """Command both servers to actually purge "deleted" files.""" - for url_purge in _urls_here_and_there(config, 'purge'): - with urlopen(Request(url_purge, method='POST')) as response: - print(f'SYNC: Calling purge via {url_purge} – {response.read()}') - - -def fill_missing(scp: SCPClient, config: Config) -> None: - """Between config.host and .remote, fill files listed in as missing.""" - missings = [] - for url_missing in _urls_here_and_there(config, 'missing'): - with urlopen(url_missing) as response: - missings += [list(json_loads(response.read()))] - with DbConn() as conn: - for i, direction_mover in enumerate([('local->remote', scp.put), - ('remote->local', scp.get)]): - direction, mover = direction_mover - for digest in (d for d in missings[i] - if d not in missings[int(not bool(i))]): - vf = VideoFile.get_one(conn, Hash.from_b64(digest)) - if vf.is_flag_set(FlagName('do not sync')): - print(f'SYNC: not sending ("do not sync" set)' - f': {vf.full_path}') - return - print(f'SYNC: sending {direction}: {vf.full_path}') - mover(vf.full_path, vf.full_path) - - -def main(): - """Connect to remote, sync local+remote DBs, + downloads where missing.""" - config = Config() - ssh = SSHClient() - ssh.load_system_host_keys() - ssh.connect(config.remote) - with SCPClient(ssh.get_transport()) as scp: - sync_dbs(scp) - purge_deleteds(config) - fill_missing(scp, config) - - -if __name__ == '__main__': - main() diff --git a/src/ytplom/db.py b/src/ytplom/db.py index f503e9b..e30e2ad 100644 --- a/src/ytplom/db.py +++ b/src/ytplom/db.py @@ -1,15 +1,19 @@ """Database access and management code.""" + +# included libs from base64 import urlsafe_b64decode, urlsafe_b64encode from hashlib import file_digest from pathlib import Path from sqlite3 import (connect as sql_connect, Connection as SqlConnection, Cursor as SqlCursor, Row as SqlRow) from typing import Callable, Literal, NewType, Optional, Self +# ourselves from ytplom.primitives import ( HandledException, NotFoundException, PATH_APP_DATA) + EXPECTED_DB_VERSION = 6 -PATH_DB = PATH_APP_DATA.joinpath('TESTdb.sql') +PATH_DB = PATH_APP_DATA.joinpath('db.sql') SqlText = NewType('SqlText', str) MigrationsList = list[tuple[Path, Optional[Callable]]] @@ -63,14 +67,12 @@ class DbFile: ) -> None: self._path = path if not path.is_file(): - raise HandledException( - f'no DB file at {path} – run "create"?') + raise HandledException(f'no DB file at {path}') if expected_version >= 0: user_version = self._get_user_version() if user_version != expected_version: raise HandledException( - f'wrong database version {user_version}, expected: ' - f'{expected_version} – run "migrate"?') + f'wrong DB version {user_version} (!= {expected_version})') def _get_user_version(self) -> int: with sql_connect(self._path) as conn: diff --git a/src/ytplom/http.py b/src/ytplom/http.py index ad2cc85..5fef79d 100644 --- a/src/ytplom/http.py +++ b/src/ytplom/http.py @@ -64,8 +64,8 @@ class Server(ThreadingHTTPServer): """Extension of parent server providing for Player and DownloadsManager.""" def __init__(self, config: Config, *args, **kwargs) -> None: - super().__init__((config.host, config.port), _TaskHandler, - *args, **kwargs) + super().__init__( + (config.host, config.port), _TaskHandler, *args, **kwargs) self.config = config self.jinja = JinjaEnv(loader=JinjaFSLoader(_PATH_TEMPLATES)) self.player = Player(config.whitelist_tags_display, @@ -416,3 +416,16 @@ class _TaskHandler(BaseHTTPRequestHandler): {'selected': 'playlist', 'filter_path': self.server.player.filter_path, 'needed_tags': self.server.player.needed_tags.joined}) + + +def serve(): + """Do Server.serve_forever on .config.port until keyboard interrupt.""" + config = Config() + server = Server(Config()) + print(f'running at port {config.port}') + try: + server.serve_forever() + except KeyboardInterrupt: + print('aborted due to keyboard interrupt; ' + 'repeat to end download thread too') + server.server_close() diff --git a/src/ytplom/migrations.py b/src/ytplom/migrations.py index 59c21f3..cb038ef 100644 --- a/src/ytplom/migrations.py +++ b/src/ytplom/migrations.py @@ -1,7 +1,10 @@ """Anything pertaining specifically to DB migrations.""" + +# included libs from pathlib import Path from typing import Callable -from ytplom.db import DbConn, MigrationsList, SqlText +# ourselves +from ytplom.db import DbConn, DbFile, MigrationsList, SqlText from ytplom.primitives import HandledException @@ -52,7 +55,7 @@ def _mig_4_convert_digests(conn: DbConn) -> None: _rewrite_files_last_field_processing_first_field(conn, bytes.fromhex) -MIGRATIONS: MigrationsList = [ +_MIGRATIONS: MigrationsList = [ (Path('0_init.sql'), None), (Path('1_add_files_last_updated.sql'), None), (Path('2_add_files_sha512.sql'), _mig_2_calc_digests), @@ -61,3 +64,8 @@ MIGRATIONS: MigrationsList = [ (Path('5_files_redo.sql'), None), (Path('6_add_files_tags.sql'), None) ] + + +def migrate(): + """Migrate DB file at expected default path to most recent version.""" + DbFile(expected_version=-1).migrate(_MIGRATIONS) diff --git a/src/ytplom/sync.py b/src/ytplom/sync.py new file mode 100644 index 0000000..89a198d --- /dev/null +++ b/src/ytplom/sync.py @@ -0,0 +1,134 @@ +"""To sync between local and remote instances.""" + +# included libs +from typing import Any, Callable +from json import loads as json_loads +from urllib.request import Request, urlopen +# non-included libs +from paramiko import SSHClient # type: ignore +from scp import SCPClient # type: ignore +# ourselves +from ytplom.db import DbConn, DbFile, Hash, PATH_DB +from ytplom.misc import (PATH_TEMP, Config, FlagName, QuotaLog, VideoFile, + YoutubeQuery, YoutubeVideo) +from ytplom.http import PAGE_NAMES + + +_PATH_DB_REMOTE = PATH_TEMP.joinpath('remote_db.sql') +_ATTR_NAME_LAST_UPDATE = 'last_update' + + +def _back_and_forth(sync_func: Callable, + dbs: tuple[DbConn, DbConn], + shared: Any + ) -> None: + """Run sync_func on arg pairs + shared, and again with pairs switched.""" + host_names = 'local', 'remote' + sync_func(host_names, dbs, shared) + sync_func(*(tuple(reversed(list(t))) for t in (host_names, dbs)), shared) + + +def _sync_objects(host_names: tuple[str, str], + dbs: tuple[DbConn, DbConn], + cls: Any + ) -> None: + """Equalize both DB's object collections; prefer newer states to older.""" + id_name = 'id_' if 'id' == cls.id_name else cls.id_name + obj_colls = cls.get_all(dbs[0]), cls.get_all(dbs[1]) + for obj_0 in [obj for obj in obj_colls[0] # only pick objs without equal + if obj not in obj_colls[1]]: # in 2nd coll, even if same ID + id_ = getattr(obj_0, id_name) + sync_down = True + to_str = obj_0 + msg_verb = 'adding' + direction = f'{host_names[1]}->{host_names[0]}' + for obj_1 in [obj for obj in obj_colls[1] # pick those from 2nd coll + if id_ == getattr(obj, id_name)]: # of same ID as obj_0 + msg_verb = 'updating' + if hasattr(obj_0, _ATTR_NAME_LAST_UPDATE): + last_update_0 = getattr(obj_0, _ATTR_NAME_LAST_UPDATE) + last_update_1 = getattr(obj_1, _ATTR_NAME_LAST_UPDATE) + sync_down = last_update_0 > last_update_1 + if not sync_down: + direction = f'{host_names[0]}->{host_names[1]}' + to_str = obj_1 + obj_1.save(dbs[0]) + if sync_down: + obj_0.save(dbs[1]) + print(f'SYNC {cls.__name__}: {msg_verb} {direction} {id_} {to_str}') + + +def _sync_relations(host_names: tuple[str, str], + dbs: tuple[DbConn, DbConn], + yt_result: YoutubeVideo + ) -> None: + """To dbs[1] add YT yt_video->yt_q_colls[0] mapping not in yt_q_colls[1]""" + yt_q_colls = tuple(YoutubeQuery.get_all_for_video(db, yt_result.id_) + for db in dbs) + direction = f'adding {host_names[1]}->{host_names[0]} mapping' + result = f'result {yt_result.id_} ({yt_result})' + for q in [q for q in yt_q_colls[0] if q not in yt_q_colls[1]]: + print(f'SYNC: {direction} of query {q.id_} ({q}) to {result}') + yt_result.save_to_query(dbs[1], q.id_) + + +def _sync_dbs(scp: SCPClient) -> None: + """Download remote DB, run sync_(objects|relations), put remote DB back.""" + scp.get(PATH_DB, _PATH_DB_REMOTE) + with DbConn(DbFile(PATH_DB).connect()) as db_local, \ + DbConn(DbFile(_PATH_DB_REMOTE).connect()) as db_remote: + for cls in (QuotaLog, YoutubeQuery, YoutubeVideo, VideoFile): + _back_and_forth(_sync_objects, (db_local, db_remote), cls) + for yt_video_local in YoutubeVideo.get_all(db_local): + _back_and_forth(_sync_relations, (db_local, db_remote), + yt_video_local) + for db in (db_remote, db_local): + db.commit() + scp.put(_PATH_DB_REMOTE, PATH_DB) + _PATH_DB_REMOTE.unlink() + + +def _urls_here_and_there(config: Config, page_name: str) -> tuple[str, ...]: + return tuple(f'http://{host}:{port}/{PAGE_NAMES[page_name]}' + for host, port in ((config.remote, config.port_remote), + (config.host, config.port))) + + +def _purge_deleteds(config: Config) -> None: + """Command both servers to actually purge "deleted" files.""" + for url_purge in _urls_here_and_there(config, 'purge'): + with urlopen(Request(url_purge, method='POST')) as response: + print(f'SYNC: Calling purge via {url_purge} – {response.read()}') + + +def _fill_missing(scp: SCPClient, config: Config) -> None: + """Between config.host and .remote, fill files listed in as missing.""" + missings = [] + for url_missing in _urls_here_and_there(config, 'missing'): + with urlopen(url_missing) as response: + missings += [list(json_loads(response.read()))] + with DbConn() as conn: + for i, direction_mover in enumerate([('local->remote', scp.put), + ('remote->local', scp.get)]): + direction, mover = direction_mover + for digest in (d for d in missings[i] + if d not in missings[int(not bool(i))]): + vf = VideoFile.get_one(conn, Hash.from_b64(digest)) + if vf.is_flag_set(FlagName('do not sync')): + print(f'SYNC: not sending ("do not sync" set)' + f': {vf.full_path}') + return + print(f'SYNC: sending {direction}: {vf.full_path}') + mover(vf.full_path, vf.full_path) + + +def sync(): + """Connect to remote, sync local+remote DBs, + downloads where missing.""" + config = Config() + ssh = SSHClient() + ssh.load_system_host_keys() + ssh.connect(config.remote) + with SCPClient(ssh.get_transport()) as scp: + _sync_dbs(scp) + _purge_deleteds(config) + _fill_missing(scp, config) diff --git a/ytplom b/ytplom index 7e5c4e2..9bea676 100755 --- a/ytplom +++ b/ytplom @@ -4,14 +4,9 @@ set -e PATH_APP_SHARE=~/.local/share/ytplom PATH_VENV="${PATH_APP_SHARE}/venv" -if [ ! "$1" = 'serve' ] && [ ! "$1" = 'sync' ] && [ ! "$1" = 'migrate' ] && [ ! "$1" = 'create' ]; then - echo "Need argument ('serve' or 'sync' or 'migrate' or 'create')." - false -fi - python3 -m venv "${PATH_VENV}" . "${PATH_VENV}/bin/activate" echo "Checking dependencies." pip3 install -r "${PATH_APP_SHARE}/requirements.txt" export PYTHONPATH="${PATH_APP_SHARE}:${PYTHONPATH}" -python3 "${PATH_APP_SHARE}/${1}.py" +python3 "${PATH_APP_SHARE}/run.py" $@