mkdir -p "${PATH_APP_SHARE}" "${PATH_LOCAL_BIN}"
-rm -f ${PATH_APP_SHARE}/migrations/*
+rm -rf ${PATH_APP_SHARE}/migrations/*
cp -r ./src/* "${PATH_APP_SHARE}/"
cp "${NAME_EXECUTABLE}" "${PATH_LOCAL_BIN}/"
#!/usr/bin/env python3
"""Script to migrate DB to most recent schema."""
+from importlib.util import spec_from_file_location, module_from_spec
+from pathlib import Path
from sys import exit as sys_exit
-from sqlite3 import connect as sql_connect
from ytplom.misc import (
EXPECTED_DB_VERSION, PATH_DB, PATH_DB_SCHEMA, PATH_MIGRATIONS,
- SQL_DB_VERSION, HandledException, get_db_version)
+ SQL_DB_VERSION, get_db_version, DbConn, HandledException, SqlText)
+
+
+_SUFFIX_PY = '.py'
+_SUFFIX_SQL = '.sql'
def main() -> None:
f'{EXPECTED_DB_VERSION}.')
print(f'Trying to migrate from DB version {start_version} to '
f'{EXPECTED_DB_VERSION} …')
- needed = [n+1 for n in range(start_version, EXPECTED_DB_VERSION)]
- migrations = {}
+ migrations: dict[int, list[Path]] = {
+ n+1: [] for n in range(start_version, EXPECTED_DB_VERSION)}
for path in [p for p in PATH_MIGRATIONS.iterdir()
if p.is_file() and p != PATH_DB_SCHEMA]:
toks = path.name.split('_')
try:
version = int(toks[0])
+ if path.suffix not in {_SUFFIX_PY, _SUFFIX_SQL}:
+ raise ValueError
except ValueError as e:
msg = f'Found illegal migration path {path}, aborting.'
raise HandledException(msg) from e
- if version in needed:
- migrations[version] = path
- missing = [n for n in needed if n not in migrations]
+ if version in migrations:
+ migrations[version] += [path]
+ missing = [n for n in migrations.keys() if not migrations[n]]
if missing:
raise HandledException(f'Needed migrations missing: {missing}')
- with sql_connect(PATH_DB) as conn:
- for version_number, migration_path in migrations.items():
- print(f'Applying migration {version_number}: {migration_path}')
- conn.executescript(migration_path.read_text(encoding='utf8'))
- conn.execute(f'{SQL_DB_VERSION} = {version_number}')
+ with DbConn(check_version=False) as conn:
+ for version, migration_paths in migrations.items():
+ sorted_paths = sorted(migration_paths)
+ msg_apply_prefix = f'Applying migration {version}: '
+ for path in [p for p in sorted_paths if _SUFFIX_SQL == p.suffix]:
+ print(f'{msg_apply_prefix}{path}')
+ sql = SqlText(path.read_text(encoding='utf8'))
+ conn.exec(sql)
+ for path in [p for p in sorted_paths if _SUFFIX_PY == p.suffix]:
+ spec = spec_from_file_location(str(path), path)
+ assert spec is not None
+ assert spec.loader is not None
+ module = module_from_spec(spec)
+ assert module is not None
+ spec.loader.exec_module(module)
+ if hasattr(module, 'migrate'):
+ print(f'{msg_apply_prefix}{path}')
+ module.migrate(conn)
+ else:
+ raise HandledException(
+ f'Suspected migration file {path} missing migrate().')
+ conn.exec(SqlText(f'{SQL_DB_VERSION} = {version}'))
+ conn.commit()
if __name__ == '__main__':
--- /dev/null
+from hashlib import file_digest
+from ytplom.misc import DbConn, HandledException, HashStr, SqlText, VideoFile
+
+
+_LEGIT_YES = 'YES!'
+
+
+def migrate(conn: DbConn) -> None:
+ file_entries = VideoFile.get_all(conn)
+ missing = [f for f in file_entries if not f.present]
+ if missing:
+ print('WARNING: Cannot find files to following paths')
+ for f in missing:
+ print(f.full_path)
+ reply = input(
+ 'WARNING: To continue migration, will have to delete above '
+ f'rows from DB. To continue, type (exactly) "{_LEGIT_YES}": ')
+ if "YES!" != reply:
+ raise HandledException('Migration aborted!')
+ for f in missing:
+ conn.exec(SqlText('DELETE FROM files WHERE rel_path = ?'),
+ (str(f.rel_path),))
+ for file in VideoFile.get_all(conn):
+ print(f'Calculating digest for: {file.rel_path}')
+ with open(file.full_path, 'rb') as x:
+ file.sha512_digest = HashStr(
+ file_digest(x, 'sha512').hexdigest())
+ file.save(conn)
--- /dev/null
+ALTER TABLE files ADD COLUMN sha512_digest TEXT NOT NULL DEFAULT "";
+++ /dev/null
-CREATE TABLE yt_queries (
- id TEXT PRIMARY KEY,
- text TEXT NOT NULL,
- retrieved_at TEXT NOT NULL
-);
-CREATE TABLE yt_videos (
- id TEXT PRIMARY KEY,
- title TEXT NOT NULL,
- description TEXT NOT NULL,
- published_at TEXT NOT NULL,
- duration TEXT NOT NULL,
- definition TEXT NOT NULL
-);
-CREATE TABLE yt_query_results (
- query_id TEXT NOT NULL,
- video_id TEXT NOT NULL,
- PRIMARY KEY (query_id, video_id),
- FOREIGN KEY (query_id) REFERENCES yt_queries(id),
- FOREIGN KEY (video_id) REFERENCES yt_videos(id)
-);
-CREATE TABLE quota_costs (
- id TEXT PRIMARY KEY,
- timestamp TEXT NOT NULL,
- cost INT NOT NULL
-);
-CREATE TABLE files (
- rel_path TEXT PRIMARY KEY,
- yt_id TEXT NOT NULL DEFAULT "",
- flags INTEGER NOT NULL DEFAULT 0,
- last_update TEXT NOT NULL DEFAULT "2000-01-01 12:00:00.123456",
- FOREIGN KEY (yt_id) REFERENCES yt_videos(id)
-);
--- /dev/null
+CREATE TABLE yt_queries (
+ id TEXT PRIMARY KEY,
+ text TEXT NOT NULL,
+ retrieved_at TEXT NOT NULL
+);
+CREATE TABLE yt_videos (
+ id TEXT PRIMARY KEY,
+ title TEXT NOT NULL,
+ description TEXT NOT NULL,
+ published_at TEXT NOT NULL,
+ duration TEXT NOT NULL,
+ definition TEXT NOT NULL
+);
+CREATE TABLE yt_query_results (
+ query_id TEXT NOT NULL,
+ video_id TEXT NOT NULL,
+ PRIMARY KEY (query_id, video_id),
+ FOREIGN KEY (query_id) REFERENCES yt_queries(id),
+ FOREIGN KEY (video_id) REFERENCES yt_videos(id)
+);
+CREATE TABLE quota_costs (
+ id TEXT PRIMARY KEY,
+ timestamp TEXT NOT NULL,
+ cost INT NOT NULL
+);
+CREATE TABLE files (
+ rel_path TEXT PRIMARY KEY,
+ yt_id TEXT NOT NULL DEFAULT "",
+ flags INTEGER NOT NULL DEFAULT 0,
+ last_update TEXT NOT NULL DEFAULT "2000-01-01 12:00:00.123456",
+ sha512_digest TEXT NOT NULL DEFAULT "",
+ FOREIGN KEY (yt_id) REFERENCES yt_videos(id)
+);
from typing import Any, Literal, NewType, Optional, Self, TypeAlias
from os import chdir, environ
from base64 import urlsafe_b64encode, urlsafe_b64decode
+from hashlib import file_digest
from random import shuffle
from time import time, sleep
from datetime import datetime, timedelta
SqlText = NewType('SqlText', str)
FlagName = NewType('FlagName', str)
FlagsInt = NewType('FlagsInt', int)
+HashStr = NewType('HashStr', str)
AmountDownloads = NewType('AmountDownloads', int)
PlayerUpdateId = NewType('PlayerUpdateId', str)
B64Str = NewType('B64Str', str)
QUOTA_COST_YOUTUBE_DETAILS = QuotaCost(1)
# database stuff
-EXPECTED_DB_VERSION = 1
+EXPECTED_DB_VERSION = 2
SQL_DB_VERSION = SqlText('PRAGMA user_version')
PATH_MIGRATIONS = PATH_APP_DATA.joinpath('migrations')
PATH_DB_SCHEMA = PATH_MIGRATIONS.joinpath(f'init_{EXPECTED_DB_VERSION}.sql')
class DbConn:
"""Wrapped sqlite3.Connection."""
- def __init__(self, path: Path = PATH_DB) -> None:
+ def __init__(self,
+ path: Path = PATH_DB,
+ check_version: bool = True
+ ) -> None:
self._path = path
if not self._path.is_file():
if self._path.exists():
with sql_connect(self._path) as conn:
conn.executescript(PATH_DB_SCHEMA.read_text(encoding='utf8'))
conn.execute(f'{SQL_DB_VERSION} = {EXPECTED_DB_VERSION}')
- cur_version = get_db_version(self._path)
- if cur_version != EXPECTED_DB_VERSION:
- raise HandledException(
- f'wrong database version {cur_version}, expected: '
- f'{EXPECTED_DB_VERSION} – run "migrate"?')
+ if check_version:
+ cur_version = get_db_version(self._path)
+ if cur_version != EXPECTED_DB_VERSION:
+ raise HandledException(
+ f'wrong database version {cur_version}, expected: '
+ f'{EXPECTED_DB_VERSION} – run "migrate"?')
self._conn = sql_connect(self._path)
def __enter__(self) -> Self:
"""Collects data about downloaded files."""
id_name = 'rel_path'
_table_name = 'files'
- _cols = ('rel_path', 'yt_id', 'flags', 'last_update')
+ _cols = ('rel_path', 'yt_id', 'flags', 'last_update', 'sha512_digest')
last_update: DatetimeStr
rel_path: Path
rel_path: Path,
yt_id: YoutubeId,
flags: FlagsInt = FlagsInt(0),
- last_update: Optional[DatetimeStr] = None
+ last_update: Optional[DatetimeStr] = None,
+ sha512_digest: Optional[HashStr] = None
) -> None:
self.rel_path = rel_path
self.yt_id = yt_id
self._renew_last_update()
else:
self.last_update = last_update
+ if sha512_digest is None:
+ with self.full_path.open('rb') as f:
+ self.sha512_digest = HashStr(
+ file_digest(f, 'sha512').hexdigest())
+ else:
+ self.sha512_digest = sha512_digest
def _renew_last_update(self):
self.last_update = DatetimeStr(datetime.now().strftime(TIMESTAMP_FMT))
for path in [p for p in Path('.').iterdir()
if p.is_file() and p not in known_paths]:
yt_id = self._id_from_filename(path)
+ print(f'SYNC: new file {path}, saving to YT ID "{yt_id}".')
file = VideoFile(path, yt_id)
- print(f'SYNC: new file {path}, saving with YT ID "{yt_id}".')
file.save(conn)
self._files = VideoFile.get_all(conn)
for file in self._files:
projects / ytplom / commitdiff