TIMESTAMP_FMT = '%Y-%m-%d %H:%M:%S.%f'
LEGAL_EXTENSIONS = {'webm', 'mp4', 'mkv'}
FILE_FLAGS: dict[FlagName, FlagsInt] = {
- FlagName('do not sync'): FlagsInt(1 << 62),
- FlagName('delete'): FlagsInt(-(1 << 63))
+ FlagName('do not sync'): FlagsInt(1 << 62)
}
ONE_MILLION = 1000 * 1000
self.last_update = _now_string()
return super().save(conn)
- @property
- def deleted(self) -> bool:
- """Return if 'delete' flag set."""
- return self.is_flag_set(FlagName('delete'))
-
- @classmethod
- def get_one(cls, conn: DbConn, id_: str | Hash) -> Self:
- """Extend super by .test_deletion."""
- file = super().get_one(conn, id_)
- if file.deleted: # pylint: disable=no-member
- raise NotFoundException('not showing entry marked as deleted')
- # NB: mypy recognizes file as VideoFile without below assert and
- # if-isinstance-else, yet less type-smart pylint only does due to the
- # latter (also the reason for the disable=no-member above; but wouldn't
- # suffice here, pylint would still identify function's return falsely);
- # the assert isn't needed by mypy and not enough for pylint, but is
- # included just so no future code change would trigger the else result.
- assert isinstance(file, VideoFile)
- return file if isinstance(file, VideoFile) else cls(None, Path(''))
-
- @classmethod
- def get_all_non_deleted(cls, conn: DbConn) -> list[Self]:
- """Extend super by excluding deleteds."""
- return [f for f in super().get_all(conn) if not f.deleted]
-
- @classmethod
- def get_all_deleted(cls, conn: DbConn) -> list[Self]:
- """Get only deleteds."""
- return [f for f in super().get_all(conn) if f.deleted]
-
@classmethod
def get_by_yt_id(cls, conn: DbConn, yt_id: YoutubeId) -> Self:
- """Return (non-deleted) VideoFile of .yt_id."""
+ """Return VideoFile of .yt_id."""
rows = conn.exec(f'SELECT * FROM {cls._table_name} WHERE yt_id =',
(yt_id,)).fetchall()
for file in [cls._from_table_row(row) for row in rows]:
- if not file.deleted:
- return file
- raise NotFoundException(
- f'no undeleted entry for file to Youtube ID {yt_id}')
+ return file
+ raise NotFoundException(f'no entry for file to Youtube ID {yt_id}')
@classmethod
def get_filtered(cls,
needed_tags_seen: TagSet = TagSet(set()),
show_absent: bool = False
) -> list[Self]:
- """Return cls.get_all_non_deleted matching provided filter criteria."""
+ """Return all matching provided filter criteria."""
return [
- f for f in cls.get_all_non_deleted(conn)
+ f for f in cls.get_all(conn)
if (show_absent or f.present)
and str(filter_path).lower() in str(f.rel_path).lower()
and (cls.tags_prefilter_needed.are_all_in(f.tags)
def all_tags_showable(cls, conn) -> TagSet:
"""Show all used tags passing .tags_display_whitelist."""
tags = TagSet()
- for file in cls.get_all_non_deleted(conn):
+ for file in cls.get_all(conn):
tags.add(file.tags.whitelisted(cls.tags_display_whitelist))
return tags
def unused_tags(self, conn: DbConn) -> TagSet:
"""Return tags used among other VideoFiles, not in self."""
tags = TagSet()
- for file in self.get_all_non_deleted(conn):
+ for file in self.get_all(conn):
tags.add(file.tags.all_not_in(self.tags).whitelisted(
self.tags_display_whitelist))
return tags
"""Return if file exists in filesystem."""
return self.full_path.exists()
- @property
- def missing(self) -> bool:
- """Return if file absent despite absence of 'delete' flag."""
- return not (self.is_flag_set(FlagName('delete')) or self.present)
-
@property
def flags_as_str_list(self) -> list[str]:
"""Return all set flags."""
"""Return if flag of flag_name is set in flags field."""
return bool(self.flags & FILE_FLAGS[flag_name])
- def ensure_unlinked_if_no_living_owners(self, conn: DbConn) -> None:
- """If 'delete' flag set and no undeleted owner, unlink."""
- if self.full_path in [f.full_path
- for f in self.get_all_non_deleted(conn)]:
- return
- if self.present:
- self.unlink_locally()
-
def unlink_locally(self) -> None:
"""Remove actual file from local filesystem."""
print(f'SYNC: removing from filesystem: {self.rel_path}')
self.full_path.unlink()
+ def purge(self, conn) -> None:
+ """Remove self from database, and in filesystem if no other owners."""
+ if self.present and self.full_path not in [
+ f.full_path for f in self.get_all(conn) if self != f]:
+ self.unlink_locally()
+ print(f'SYNC: purging off DB: {self.digest.b64} ({self.rel_path})')
+ conn.exec(f'DELETE FROM {self._table_name} WHERE digest =',
+ (self.digest.bytes,))
+
@classmethod
- def purge_deleteds(cls, conn: DbConn) -> None:
- """For all of .is_flag_set("deleted"), remove file _and_ DB entry."""
- for file in cls.get_all_deleted(conn):
- if file.present:
- file.unlink_locally()
- print(f'SYNC: purging off DB: {file.digest.b64} ({file.rel_path})')
- conn.exec(f'DELETE FROM {cls._table_name} WHERE digest =',
- (file.digest.bytes,))
+ def purge_deleteds(cls, conn: DbConn) -> list[Hash]:
+ """Purge all with .last_update < a respective entry in delete table."""
+ too_early = '2000-01-01 00:00:00.0'
+ del_req_by_timestamp: dict[Hash, DatetimeStr] = {}
+ for del_req in FileDeletionRequest.get_all(conn):
+ if del_req_by_timestamp.get(del_req.digest, too_early
+ ) < del_req.last_update:
+ del_req_by_timestamp[del_req.digest] = del_req.last_update
+ deleteds: list[Hash] = []
+ for file in [file for file in cls.get_all(conn)
+ if del_req_by_timestamp.get(file.digest, too_early
+ ) > file.last_update]:
+ deleteds += [file.digest]
+ file.purge(conn)
+ return deleteds
+
+ def delete(self, conn: DbConn) -> None:
+ """Add/update into deletion request table, then purge locally."""
+ FileDeletionRequest(self.digest, _now_string()).save(conn)
+ self.purge(conn)
+
+
+class FileDeletionRequest(DbData):
+ """Collect file deletion requests in their own table, for syncability."""
+ id_name = 'digest'
+ _table_name = 'file_deletion_requests'
+ _cols = ('digest', 'last_update')
+ digest: Hash
+ last_update: DatetimeStr
+
+ def __init__(self, digest: Hash, last_update: DatetimeStr) -> None:
+ self.digest = digest
+ self.last_update = last_update
+
+ def __str__(self) -> str:
+ return f'{self.digest.b64}:{self.last_update}'
class QuotaLog(DbData):
def _sync_db(self):
with DbConn() as conn:
- known_paths = [file.rel_path for
- file in VideoFile.get_all_non_deleted(conn)]
+ VideoFile.purge_deleteds(conn)
+ known_paths = [file.rel_path for file in VideoFile.get_all(conn)]
old_cwd = Path.cwd()
chdir(PATH_DOWNLOADS)
for path in [p for p in Path('.').iterdir() if p.is_file()]:
'present',
str(path),
VideoFile.get_by_yt_id(conn, yt_id).digest.b64)
- for file in VideoFile.get_all_deleted(conn):
- file.ensure_unlinked_if_no_living_owners(conn)
- self._files = VideoFile.get_all_non_deleted(conn)
+ self._files = VideoFile.get_all(conn)
chdir(old_cwd)
def last_update_for(self,
from scp import SCPClient # type: ignore
# ourselves
from ytplom.db import DbConn, DbFile, Hash, PATH_DB
-from ytplom.misc import (PATH_TEMP, Config, FlagName, QuotaLog, VideoFile,
- YoutubeQuery, YoutubeVideo)
+from ytplom.misc import (PATH_TEMP, Config, FileDeletionRequest, FlagName,
+ QuotaLog, VideoFile, YoutubeQuery, YoutubeVideo)
from ytplom.http import PAGE_NAMES
"""Download remote DB, run sync_(objects|relations), put remote DB back."""
scp.get(PATH_DB, _PATH_DB_REMOTE)
with DbConn() as db_local, DbConn(DbFile(_PATH_DB_REMOTE)) as db_remote:
- for cls in (QuotaLog, YoutubeQuery, YoutubeVideo, VideoFile):
+ for cls in (FileDeletionRequest, QuotaLog, YoutubeQuery, YoutubeVideo,
+ VideoFile):
_back_and_forth(_sync_objects, (db_local, db_remote), cls)
for yt_video_local in YoutubeVideo.get_all(db_local):
_back_and_forth(_sync_relations, (db_local, db_remote),
projects / ytplom / commitdiff