+++ /dev/null
-#!/usr/bin/env python3
-"""Script to migrate DB to most recent schema."""
-from ytplom.db import DbFile
-from ytplom.migrations import MIGRATIONS
-
-
-if __name__ == '__main__':
- DbFile(expected_version=-1).migrate(MIGRATIONS)
--- /dev/null
+#!/usr/bin/env python3
+"""Match command line calls to appropriate scripts."""
+
+# included libs
+from sys import argv, exit as sys_exit
+# ourselves
+from ytplom.db import DbFile
+from ytplom.primitives import HandledException
+from ytplom.migrations import migrate
+from ytplom.http import serve
+from ytplom.sync import sync
+
+
+if __name__ == '__main__':
+ try:
+ if 2 != len(argv):
+ raise HandledException('Bad number of command arguments.')
+ match argv[1]:
+ case 'create_db':
+ DbFile.create()
+ case 'migrate_db':
+ migrate()
+ case 'serve':
+ serve()
+ case 'sync':
+ sync()
+ case _:
+ raise HandledException('Unknown argument.')
+ except HandledException as e:
+ print(e)
+ sys_exit(1)
+++ /dev/null
-#!/usr/bin/env python3
-"""Minimalistic download-focused YouTube interface."""
-from ytplom.misc import Config
-from ytplom.http import Server
-
-
-if __name__ == '__main__':
- config = Config()
- server = Server(config)
- print(f'running at port {config.port}')
- try:
- server.serve_forever()
- except KeyboardInterrupt:
- print('aborted due to keyboard interrupt; '
- 'repeat to end download thread too')
- server.server_close()
+++ /dev/null
-#!/usr/bin/env python3
-"""Script to sync between local and remote instances."""
-
-# included libs
-from typing import Any, Callable
-from json import loads as json_loads
-from urllib.request import Request, urlopen
-# non-included libs
-from paramiko import SSHClient # type: ignore
-from scp import SCPClient # type: ignore
-from ytplom.db import DbConn, DbFile, Hash, PATH_DB
-from ytplom.misc import (PATH_TEMP, Config, FlagName, QuotaLog, VideoFile,
- YoutubeQuery, YoutubeVideo)
-from ytplom.http import PAGE_NAMES
-
-
-PATH_DB_REMOTE = PATH_TEMP.joinpath('remote_db.sql')
-ATTR_NAME_LAST_UPDATE = 'last_update'
-
-
-def back_and_forth(sync_func: Callable,
- dbs: tuple[DbConn, DbConn],
- shared: Any
- ) -> None:
- """Run sync_func on arg pairs + shared, and again with pairs switched."""
- host_names = 'local', 'remote'
- sync_func(host_names, dbs, shared)
- sync_func(*(tuple(reversed(list(t))) for t in (host_names, dbs)), shared)
-
-
-def sync_objects(host_names: tuple[str, str],
- dbs: tuple[DbConn, DbConn],
- cls: Any
- ) -> None:
- """Equalize both DB's object collections; prefer newer states to older."""
- id_name = 'id_' if 'id' == cls.id_name else cls.id_name
- obj_colls = cls.get_all(dbs[0]), cls.get_all(dbs[1])
- for obj_0 in [obj for obj in obj_colls[0] # only pick objs without equal
- if obj not in obj_colls[1]]: # in 2nd coll, even if same ID
- id_ = getattr(obj_0, id_name)
- sync_down = True
- to_str = obj_0
- msg_verb = 'adding'
- direction = f'{host_names[1]}->{host_names[0]}'
- for obj_1 in [obj for obj in obj_colls[1] # pick those from 2nd coll
- if id_ == getattr(obj, id_name)]: # of same ID as obj_0
- msg_verb = 'updating'
- if hasattr(obj_0, ATTR_NAME_LAST_UPDATE):
- last_update_0 = getattr(obj_0, ATTR_NAME_LAST_UPDATE)
- last_update_1 = getattr(obj_1, ATTR_NAME_LAST_UPDATE)
- sync_down = last_update_0 > last_update_1
- if not sync_down:
- direction = f'{host_names[0]}->{host_names[1]}'
- to_str = obj_1
- obj_1.save(dbs[0])
- if sync_down:
- obj_0.save(dbs[1])
- print(f'SYNC {cls.__name__}: {msg_verb} {direction} {id_} {to_str}')
-
-
-def sync_relations(host_names: tuple[str, str],
- dbs: tuple[DbConn, DbConn],
- yt_result: YoutubeVideo
- ) -> None:
- """To dbs[1] add YT yt_video->yt_q_colls[0] mapping not in yt_q_colls[1]"""
- yt_q_colls = tuple(YoutubeQuery.get_all_for_video(db, yt_result.id_)
- for db in dbs)
- direction = f'adding {host_names[1]}->{host_names[0]} mapping'
- result = f'result {yt_result.id_} ({yt_result})'
- for q in [q for q in yt_q_colls[0] if q not in yt_q_colls[1]]:
- print(f'SYNC: {direction} of query {q.id_} ({q}) to {result}')
- yt_result.save_to_query(dbs[1], q.id_)
-
-
-def sync_dbs(scp: SCPClient) -> None:
- """Download remote DB, run sync_(objects|relations), put remote DB back."""
- scp.get(PATH_DB, PATH_DB_REMOTE)
- with DbConn(DbFile(PATH_DB).connect()) as db_local, \
- DbConn(DbFile(PATH_DB_REMOTE).connect()) as db_remote:
- for cls in (QuotaLog, YoutubeQuery, YoutubeVideo, VideoFile):
- back_and_forth(sync_objects, (db_local, db_remote), cls)
- for yt_video_local in YoutubeVideo.get_all(db_local):
- back_and_forth(sync_relations, (db_local, db_remote),
- yt_video_local)
- for db in (db_remote, db_local):
- db.commit()
- scp.put(PATH_DB_REMOTE, PATH_DB)
- PATH_DB_REMOTE.unlink()
-
-
-def _urls_here_and_there(config: Config, page_name: str) -> tuple[str, ...]:
- return tuple(f'http://{host}:{port}/{PAGE_NAMES[page_name]}'
- for host, port in ((config.remote, config.port_remote),
- (config.host, config.port)))
-
-
-def purge_deleteds(config: Config) -> None:
- """Command both servers to actually purge "deleted" files."""
- for url_purge in _urls_here_and_there(config, 'purge'):
- with urlopen(Request(url_purge, method='POST')) as response:
- print(f'SYNC: Calling purge via {url_purge} – {response.read()}')
-
-
-def fill_missing(scp: SCPClient, config: Config) -> None:
- """Between config.host and .remote, fill files listed in as missing."""
- missings = []
- for url_missing in _urls_here_and_there(config, 'missing'):
- with urlopen(url_missing) as response:
- missings += [list(json_loads(response.read()))]
- with DbConn() as conn:
- for i, direction_mover in enumerate([('local->remote', scp.put),
- ('remote->local', scp.get)]):
- direction, mover = direction_mover
- for digest in (d for d in missings[i]
- if d not in missings[int(not bool(i))]):
- vf = VideoFile.get_one(conn, Hash.from_b64(digest))
- if vf.is_flag_set(FlagName('do not sync')):
- print(f'SYNC: not sending ("do not sync" set)'
- f': {vf.full_path}')
- return
- print(f'SYNC: sending {direction}: {vf.full_path}')
- mover(vf.full_path, vf.full_path)
-
-
-def main():
- """Connect to remote, sync local+remote DBs, + downloads where missing."""
- config = Config()
- ssh = SSHClient()
- ssh.load_system_host_keys()
- ssh.connect(config.remote)
- with SCPClient(ssh.get_transport()) as scp:
- sync_dbs(scp)
- purge_deleteds(config)
- fill_missing(scp, config)
-
-
-if __name__ == '__main__':
- main()
"""Database access and management code."""
+
+# included libs
from base64 import urlsafe_b64decode, urlsafe_b64encode
from hashlib import file_digest
from pathlib import Path
from sqlite3 import (connect as sql_connect, Connection as SqlConnection,
Cursor as SqlCursor, Row as SqlRow)
from typing import Callable, Literal, NewType, Optional, Self
+# ourselves
from ytplom.primitives import (
HandledException, NotFoundException, PATH_APP_DATA)
+
EXPECTED_DB_VERSION = 6
-PATH_DB = PATH_APP_DATA.joinpath('TESTdb.sql')
+PATH_DB = PATH_APP_DATA.joinpath('db.sql')
SqlText = NewType('SqlText', str)
MigrationsList = list[tuple[Path, Optional[Callable]]]
) -> None:
self._path = path
if not path.is_file():
- raise HandledException(
- f'no DB file at {path} – run "create"?')
+ raise HandledException(f'no DB file at {path}')
if expected_version >= 0:
user_version = self._get_user_version()
if user_version != expected_version:
raise HandledException(
- f'wrong database version {user_version}, expected: '
- f'{expected_version} – run "migrate"?')
+ f'wrong DB version {user_version} (!= {expected_version})')
def _get_user_version(self) -> int:
with sql_connect(self._path) as conn:
"""Extension of parent server providing for Player and DownloadsManager."""
def __init__(self, config: Config, *args, **kwargs) -> None:
- super().__init__((config.host, config.port), _TaskHandler,
- *args, **kwargs)
+ super().__init__(
+ (config.host, config.port), _TaskHandler, *args, **kwargs)
self.config = config
self.jinja = JinjaEnv(loader=JinjaFSLoader(_PATH_TEMPLATES))
self.player = Player(config.whitelist_tags_display,
{'selected': 'playlist',
'filter_path': self.server.player.filter_path,
'needed_tags': self.server.player.needed_tags.joined})
+
+
+def serve():
+ """Do Server.serve_forever on .config.port until keyboard interrupt."""
+ config = Config()
+ server = Server(Config())
+ print(f'running at port {config.port}')
+ try:
+ server.serve_forever()
+ except KeyboardInterrupt:
+ print('aborted due to keyboard interrupt; '
+ 'repeat to end download thread too')
+ server.server_close()
"""Anything pertaining specifically to DB migrations."""
+
+# included libs
from pathlib import Path
from typing import Callable
-from ytplom.db import DbConn, MigrationsList, SqlText
+# ourselves
+from ytplom.db import DbConn, DbFile, MigrationsList, SqlText
from ytplom.primitives import HandledException
_rewrite_files_last_field_processing_first_field(conn, bytes.fromhex)
-MIGRATIONS: MigrationsList = [
+_MIGRATIONS: MigrationsList = [
(Path('0_init.sql'), None),
(Path('1_add_files_last_updated.sql'), None),
(Path('2_add_files_sha512.sql'), _mig_2_calc_digests),
(Path('5_files_redo.sql'), None),
(Path('6_add_files_tags.sql'), None)
]
+
+
+def migrate():
+ """Migrate DB file at expected default path to most recent version."""
+ DbFile(expected_version=-1).migrate(_MIGRATIONS)
--- /dev/null
+"""To sync between local and remote instances."""
+
+# included libs
+from typing import Any, Callable
+from json import loads as json_loads
+from urllib.request import Request, urlopen
+# non-included libs
+from paramiko import SSHClient # type: ignore
+from scp import SCPClient # type: ignore
+# ourselves
+from ytplom.db import DbConn, DbFile, Hash, PATH_DB
+from ytplom.misc import (PATH_TEMP, Config, FlagName, QuotaLog, VideoFile,
+ YoutubeQuery, YoutubeVideo)
+from ytplom.http import PAGE_NAMES
+
+
+_PATH_DB_REMOTE = PATH_TEMP.joinpath('remote_db.sql')
+_ATTR_NAME_LAST_UPDATE = 'last_update'
+
+
+def _back_and_forth(sync_func: Callable,
+ dbs: tuple[DbConn, DbConn],
+ shared: Any
+ ) -> None:
+ """Run sync_func on arg pairs + shared, and again with pairs switched."""
+ host_names = 'local', 'remote'
+ sync_func(host_names, dbs, shared)
+ sync_func(*(tuple(reversed(list(t))) for t in (host_names, dbs)), shared)
+
+
+def _sync_objects(host_names: tuple[str, str],
+ dbs: tuple[DbConn, DbConn],
+ cls: Any
+ ) -> None:
+ """Equalize both DB's object collections; prefer newer states to older."""
+ id_name = 'id_' if 'id' == cls.id_name else cls.id_name
+ obj_colls = cls.get_all(dbs[0]), cls.get_all(dbs[1])
+ for obj_0 in [obj for obj in obj_colls[0] # only pick objs without equal
+ if obj not in obj_colls[1]]: # in 2nd coll, even if same ID
+ id_ = getattr(obj_0, id_name)
+ sync_down = True
+ to_str = obj_0
+ msg_verb = 'adding'
+ direction = f'{host_names[1]}->{host_names[0]}'
+ for obj_1 in [obj for obj in obj_colls[1] # pick those from 2nd coll
+ if id_ == getattr(obj, id_name)]: # of same ID as obj_0
+ msg_verb = 'updating'
+ if hasattr(obj_0, _ATTR_NAME_LAST_UPDATE):
+ last_update_0 = getattr(obj_0, _ATTR_NAME_LAST_UPDATE)
+ last_update_1 = getattr(obj_1, _ATTR_NAME_LAST_UPDATE)
+ sync_down = last_update_0 > last_update_1
+ if not sync_down:
+ direction = f'{host_names[0]}->{host_names[1]}'
+ to_str = obj_1
+ obj_1.save(dbs[0])
+ if sync_down:
+ obj_0.save(dbs[1])
+ print(f'SYNC {cls.__name__}: {msg_verb} {direction} {id_} {to_str}')
+
+
+def _sync_relations(host_names: tuple[str, str],
+ dbs: tuple[DbConn, DbConn],
+ yt_result: YoutubeVideo
+ ) -> None:
+ """To dbs[1] add YT yt_video->yt_q_colls[0] mapping not in yt_q_colls[1]"""
+ yt_q_colls = tuple(YoutubeQuery.get_all_for_video(db, yt_result.id_)
+ for db in dbs)
+ direction = f'adding {host_names[1]}->{host_names[0]} mapping'
+ result = f'result {yt_result.id_} ({yt_result})'
+ for q in [q for q in yt_q_colls[0] if q not in yt_q_colls[1]]:
+ print(f'SYNC: {direction} of query {q.id_} ({q}) to {result}')
+ yt_result.save_to_query(dbs[1], q.id_)
+
+
+def _sync_dbs(scp: SCPClient) -> None:
+ """Download remote DB, run sync_(objects|relations), put remote DB back."""
+ scp.get(PATH_DB, _PATH_DB_REMOTE)
+ with DbConn(DbFile(PATH_DB).connect()) as db_local, \
+ DbConn(DbFile(_PATH_DB_REMOTE).connect()) as db_remote:
+ for cls in (QuotaLog, YoutubeQuery, YoutubeVideo, VideoFile):
+ _back_and_forth(_sync_objects, (db_local, db_remote), cls)
+ for yt_video_local in YoutubeVideo.get_all(db_local):
+ _back_and_forth(_sync_relations, (db_local, db_remote),
+ yt_video_local)
+ for db in (db_remote, db_local):
+ db.commit()
+ scp.put(_PATH_DB_REMOTE, PATH_DB)
+ _PATH_DB_REMOTE.unlink()
+
+
+def _urls_here_and_there(config: Config, page_name: str) -> tuple[str, ...]:
+ return tuple(f'http://{host}:{port}/{PAGE_NAMES[page_name]}'
+ for host, port in ((config.remote, config.port_remote),
+ (config.host, config.port)))
+
+
+def _purge_deleteds(config: Config) -> None:
+ """Command both servers to actually purge "deleted" files."""
+ for url_purge in _urls_here_and_there(config, 'purge'):
+ with urlopen(Request(url_purge, method='POST')) as response:
+ print(f'SYNC: Calling purge via {url_purge} – {response.read()}')
+
+
+def _fill_missing(scp: SCPClient, config: Config) -> None:
+ """Between config.host and .remote, fill files listed in as missing."""
+ missings = []
+ for url_missing in _urls_here_and_there(config, 'missing'):
+ with urlopen(url_missing) as response:
+ missings += [list(json_loads(response.read()))]
+ with DbConn() as conn:
+ for i, direction_mover in enumerate([('local->remote', scp.put),
+ ('remote->local', scp.get)]):
+ direction, mover = direction_mover
+ for digest in (d for d in missings[i]
+ if d not in missings[int(not bool(i))]):
+ vf = VideoFile.get_one(conn, Hash.from_b64(digest))
+ if vf.is_flag_set(FlagName('do not sync')):
+ print(f'SYNC: not sending ("do not sync" set)'
+ f': {vf.full_path}')
+ return
+ print(f'SYNC: sending {direction}: {vf.full_path}')
+ mover(vf.full_path, vf.full_path)
+
+
+def sync():
+ """Connect to remote, sync local+remote DBs, + downloads where missing."""
+ config = Config()
+ ssh = SSHClient()
+ ssh.load_system_host_keys()
+ ssh.connect(config.remote)
+ with SCPClient(ssh.get_transport()) as scp:
+ _sync_dbs(scp)
+ _purge_deleteds(config)
+ _fill_missing(scp, config)
PATH_APP_SHARE=~/.local/share/ytplom
PATH_VENV="${PATH_APP_SHARE}/venv"
-if [ ! "$1" = 'serve' ] && [ ! "$1" = 'sync' ] && [ ! "$1" = 'migrate' ] && [ ! "$1" = 'create' ]; then
- echo "Need argument ('serve' or 'sync' or 'migrate' or 'create')."
- false
-fi
-
python3 -m venv "${PATH_VENV}"
. "${PATH_VENV}/bin/activate"
echo "Checking dependencies."
pip3 install -r "${PATH_APP_SHARE}/requirements.txt"
export PYTHONPATH="${PATH_APP_SHARE}:${PYTHONPATH}"
-python3 "${PATH_APP_SHARE}/${1}.py"
+python3 "${PATH_APP_SHARE}/run.py" $@
projects / ytplom / commitdiff