From: Christian Heller Date: Fri, 14 Nov 2025 08:15:57 +0000 (+0100) Subject: Add "downloads" page to observe and manipulate downloads queue. X-Git-Url: https://plomlompom.com/repos/%7B%7Bitem_name%7D%7D?a=commitdiff_plain;h=3065ef4c1f74f80d4d09590725670058dad4b2ed;p=ytplom Add "downloads" page to observe and manipulate downloads queue. --- diff --git a/src/templates/_base.tmpl b/src/templates/_base.tmpl index 8764a5e..1845b4a 100644 --- a/src/templates/_base.tmpl +++ b/src/templates/_base.tmpl @@ -163,6 +163,7 @@ td, th { vertical-align: top; text-align: left; margin: 0; padding: 0; } · {{ macros.link_if("files" != selected, page_names.files) }} · {{ macros.link_if("tags" != selected, page_names.tags) }} · {{ macros.link_if("yt_queries" != selected, page_names.yt_queries, "queries") }} +· {{ macros.link_if("downloads" != selected, page_names.downloads) }}
diff --git a/src/templates/downloads.tmpl b/src/templates/downloads.tmpl new file mode 100644 index 0000000..2ed03f8 --- /dev/null +++ b/src/templates/downloads.tmpl @@ -0,0 +1,125 @@ +{% extends '_base.tmpl' %} + + +{% block script %} +events_params += 'downloads=1'; +event_handlers.downloads = function(data) { + let table = document.getElementById('downloaded_rows'); + table.innerHTML = ''; + for (let i = 0; i < data.downloaded.length; i++) { + const download_ = data.downloaded[i]; + const tr = add_child_to( + tag='tr', + parent=table); + add_a_to( + parent=add_child_to( + tag='td', + parent=tr), + text=download_.path, + href="/{{page_names.file}}/" + download_.digest); + add_a_to( + parent=add_child_to( + tag='td', + parent=tr), + text=download_.title, + href="/{{page_names.yt_result}}/" + download_.yt_id); + } + + add_child_to( + tag='td', + parent=add_child_to( + tag='tr', + parent=table), + attrs={ + textContent: 'DOWNLOADING'}); + + if (data.downloading) { + const tr = add_child_to( + tag='tr', + parent=table); + // col 1 + const td_entry_control = add_child_to( + tag='td', + parent=tr); + add_child_to( + tag='button', + parent=td_entry_control, + attrs={ + textContent: 'x', + onclick: function() { + wrapped_post( + target="{{page_names.downloads_json}}", + body={ + command: ['abort']}); } + }) + // col 2 + add_child_to( + tag='td', + parent=tr, + attrs={ + textContent: data.downloading.status}); + // col 3 + add_a_to( + parent=add_child_to( + tag='td', + parent=tr), + text=data.downloading.title, + href="/{{page_names.yt_result}}/" + data.downloading.yt_id); + } + + for (let i = 0; i < data.to_download.length; i++) { + const download_ = data.to_download[i]; + if (download_.status == 'downloaded') { + continue; + } + const tr = add_child_to( + tag='tr', + parent=table); + // col 1 + add_child_to( + tag='button', + parent=tr, + attrs={ + textContent: 'x', + onclick: function() { + wrapped_post( + target="{{page_names.downloads_json}}", + body={ + command: [`unqueue_${i}`]}); } + }) + // col 2 + const td_entry_control = add_child_to( + tag='td', + parent=tr); + for (const [symbol, prefix] of [['^', 'up'], + ['v', 'down']]) { + add_child_to( + tag='button', + parent=td_entry_control, + attrs={ + textContent: symbol, + onclick: function() { + wrapped_post( + target="{{page_names.downloads_json}}", + body={ + command: [`${prefix}_${i}`]}); } + }) + } + // col 3 + add_a_to( + parent=add_child_to( + tag='td', + parent=tr), + text=download_.title, + href="/{{page_names.yt_result}}/" + download_.yt_id); + } +} +{% endblock %} + + +{% block body %} + +
+ +
+{% endblock %} diff --git a/src/templates/yt_result.tmpl b/src/templates/yt_result.tmpl index 2aa3737..fe68a43 100644 --- a/src/templates/yt_result.tmpl +++ b/src/templates/yt_result.tmpl @@ -7,12 +7,19 @@ event_handlers.download = function(data) { const td = document.getElementById("status"); td.innerHTML = ""; if ("absent" == data.status) { - add_a_to(td, "download?", "/{{page_names.download}}/{{video_data.id_}}"); + add_a_to( + parent=td, + text="download?", + href="/{{page_names.download}}/{{video_data.id_}}"); } else if ("present" == data.status) { - add_a_to(td, data.path, "/{{page_names.file}}/" + data.digest); + add_a_to( + parent=td, + text=data.path, + href="/{{page_names.file}}/" + data.digest); } else { - add_text_to(td, `${data.status}`); - } + add_text_to( + parent=td, + text=`${data.status}`); } } {% endblock %} diff --git a/src/ytplom/http.py b/src/ytplom/http.py index c7fe804..2eb7930 100644 --- a/src/ytplom/http.py +++ b/src/ytplom/http.py @@ -26,6 +26,7 @@ _THUMBNAIL_URL_SUFFIX = '/default.jpg' # template paths _PATH_TEMPLATES = PATH_APP_DATA.joinpath('templates') +_NAME_TEMPLATE_DOWNLOADS = Path('downloads.tmpl') _NAME_TEMPLATE_FILE_DATA = Path('file_data.tmpl') _NAME_TEMPLATE_FILES = Path('files.tmpl') _NAME_TEMPLATE_PLAYLIST = Path('playlist.tmpl') @@ -37,6 +38,8 @@ _NAME_TEMPLATE_YT_RESULTS = Path('yt_results.tmpl') # page names PAGE_NAMES: dict[str, str] = { 'download': 'dl', + 'downloads': 'downloads', + 'downloads_json': 'downloads.json', 'events': 'events', 'file': 'file', 'file_kill': 'file_kill', @@ -108,7 +111,9 @@ class _TaskHandler(PlomHttpHandler): def do_POST(self) -> None: # pylint:disable=invalid-name """Map POST requests to handlers for various paths.""" - if self.pagename == PAGE_NAMES['file_json']: + if self.pagename == PAGE_NAMES['downloads_json']: + self._receive_downloads() + elif self.pagename == PAGE_NAMES['file_json']: self._update_file() elif self.pagename == PAGE_NAMES['file_kill']: self._kill_file() @@ -119,6 +124,10 @@ class _TaskHandler(PlomHttpHandler): elif self.pagename == PAGE_NAMES['yt_queries']: self._receive_yt_query() + def _receive_downloads(self) -> None: + self.server.downloads.q.put(self.postvars.first_for('command')) + self.send_http(b'OK') + def _kill_file(self) -> None: if not self.server.config.allow_file_edit: self.send_http(b'no way', code=403) @@ -197,6 +206,8 @@ class _TaskHandler(PlomHttpHandler): try: if self.pagename == PAGE_NAMES['download']: self._send_or_download_video() + elif self.pagename == PAGE_NAMES['downloads']: + self._send_downloads() elif self.pagename == PAGE_NAMES['events']: self._send_events() elif self.pagename == PAGE_NAMES['file']: @@ -253,12 +264,15 @@ class _TaskHandler(PlomHttpHandler): .joinpath(PAGE_NAMES['yt_result']) .joinpath(video_id)) + def _send_downloads(self) -> None: + self._send_rendered_template(_NAME_TEMPLATE_DOWNLOADS, {}) + def _send_events(self) -> None: self.send_http(headers=[(_HEADER_CONTENT_TYPE, 'text/event-stream'), ('Cache-Control', 'no-cache'), ('Connection', 'keep-alive')]) selected: Optional[VideoFile] = None - last_updates = {'player': '', 'download': ''} + last_updates = {'player': '', 'download': '', 'downloads': ''} payload: dict[str, Any] = {} time_last_write = 0.0 subscriptions = ['player'] + list(self.params.as_dict.keys()) @@ -319,6 +333,12 @@ class _TaskHandler(PlomHttpHandler): last_updates['download'] = update['time'] payload['download'] = {k: update[k] for k in update if k != 'time'} + if 'downloads' in subscriptions: + if last_updates['downloads'] < self.server.downloads.timestamp: + last_updates['downloads'] = self.server.downloads.timestamp + with DbConn() as conn: + payload['downloads']\ + = self.server.downloads.overview(conn) if not payload: sleep(_EVENTS_UPDATE_INTERVAL_S) diff --git a/src/ytplom/misc.py b/src/ytplom/misc.py index 760d0b4..33a4a9b 100644 --- a/src/ytplom/misc.py +++ b/src/ytplom/misc.py @@ -837,6 +837,7 @@ class DownloadsManager: """Manages downloading and downloads access.""" def __init__(self) -> None: + self._inherited: list[YoutubeId] = [] self._downloaded: list[YoutubeId] = [] self._downloading: Optional[YoutubeId] = None self._to_download: list[YoutubeId] = [] @@ -861,16 +862,22 @@ class DownloadsManager: with DbConn() as conn: file.save(conn) conn.commit() - self._downloaded += [yt_id] + self._inherited += [yt_id] + self._update_timestamp(yt_id) + self._timestamp: DatetimeStr = _now_string() chdir(old_cwd) + @property + def timestamp(self) -> DatetimeStr: + 'When most recent change has occured in downloads management.' + return max(self._timestamp, *self._timestamps.values()) + def last_update_for(self, conn: DbConn, yt_id: YoutubeId ) -> dict[str, str]: 'For yt_id construct update with timestamp, status, optional fields.' update = { 'time': self._timestamps.get(yt_id, '2000-01-01'), - ## 'status': ('present' if yt_id in self._downloaded + self._inherited - 'status': ('present' if yt_id in self._downloaded + 'status': ('present' if yt_id in self._downloaded + self._inherited else ('queued' if yt_id in self._to_download else (self._status if yt_id == self._downloading else 'absent')))} @@ -883,6 +890,43 @@ class DownloadsManager: def _update_timestamp(self, yt_id: YoutubeId) -> None: self._timestamps[yt_id] = _now_string() + def overview( + self, conn: DbConn + ) -> dict[str, list[dict[str, str]] | Optional[dict[str, str]]]: + 'What has been, what will be, and what currently is being downloaded.' + downloaded: list[dict[str, str]] = [] + yt_id: Optional[YoutubeId] + for yt_id in self._downloaded: + try: + file = VideoFile.get_by_yt_id(conn, yt_id) + except NotFoundException: + continue + downloaded += [{'yt_id': yt_id, + 'path': str(file.rel_path), + 'digest': file.digest.b64}] # + title? + to_download: list[dict[str, str]] = [] + for idx, yt_id in enumerate(self._to_download): + try: + yt_video = YoutubeVideo.get_one(conn, yt_id) + except NotFoundException: + continue + to_download += [{'yt_id': yt_id, + 'title': yt_video.title, + 'idx': str(idx)}] + downloading: Optional[dict[str, str]] = None + if (yt_id := self._downloading): + try: + yt_video = YoutubeVideo.get_one(conn, yt_id) + except NotFoundException: + pass + else: + downloading = {'yt_id': yt_id, + 'title': yt_video.title, + 'status': self._status} + return {'downloaded': downloaded, + 'downloading': downloading, + 'to_download': to_download} + @staticmethod def _id_from_filename(path: Path) -> YoutubeId: return YoutubeId(path.stem.split('[')[-1].split(']')[0]) @@ -900,9 +944,26 @@ class DownloadsManager: if not self._downloading: self.q.put('download_next') + def _unqueue_download(self, idx: int) -> YoutubeId: + yt_id = self._to_download[idx] + self._to_download.remove(yt_id) + return yt_id + def _forget_file(self, yt_id: YoutubeId) -> None: - if yt_id in self._downloaded: - self._downloaded.remove(yt_id) + for attr_name in ('_downloaded', '_inherited'): + coll = getattr(self, attr_name) + if yt_id in coll: + # NB: for some strange remove .remove on target + # collection will not satisfy assert below … + setattr(self, attr_name, [id_ for id_ in coll if id_ != yt_id]) + assert yt_id not in self._downloaded + self._inherited + + def _abort_downloading(self) -> YoutubeId: + target = self._downloading + self._downloading = None + assert target is not None + self.q.put('download_next') + return target def _savefile(self, arg: str) -> YoutubeId: filename = Path(arg) @@ -918,6 +979,13 @@ class DownloadsManager: self._downloading = None return yt_id + def _move_in_queue(self, start_idx: int, upwards: bool) -> None: + i0, i1 = start_idx, start_idx + (-1 if upwards else 1) + if i1 < 0 or i1 >= len(self._to_download): + return + self._to_download[i0], self._to_download[i1] = (self._to_download[i1], + self._to_download[i0]) + def _download_next(self) -> None: if not self._to_download: return @@ -926,9 +994,14 @@ class DownloadsManager: self._downloading = yt_id = self._to_download.pop(0) filename: Optional[str] = None + class AbortedException(Exception): + 'To signal abortion of download has been requested.' + def hook(d) -> None: nonlocal downloaded_before nonlocal filename + if self._downloading != yt_id: + raise AbortedException() if d['status'] in {'downloading', 'finished'}: downloaded_i = d[TOK_LOADED] downloaded_mb = (downloaded_i + downloaded_before) / MEGA @@ -953,6 +1026,8 @@ class DownloadsManager: try: self._status = 'extracting download info' info_dirty = ydl.extract_info(url, download=False) + if self._downloading != yt_id: + raise AbortedException() info = ydl.sanitize_info(info_dirty) key_formats = 'requested_formats' if key_formats not in info: @@ -962,9 +1037,10 @@ class DownloadsManager: if TOK_FS_AP in f: sizes[f[TOK_FO_ID]] = [True, f[TOK_FS_AP]] ydl.download(url) - except YoutubeDLError as e: + except (YoutubeDLError, AbortedException) as e: self._update_timestamp(yt_id) - self._status = 'ERROR' + if isinstance(e, YoutubeDLError): + self._status = 'ERROR' raise e self.q.put(f'savefile_{filename}') self.q.put('download_next') @@ -973,19 +1049,30 @@ class DownloadsManager: 'Collect, enact commands sent through .q.' def loop() -> None: while True: + self._timestamp = _now_string() command = self.q.get() if command == 'download_next': Thread(target=self._download_next, daemon=False).start() continue - command, arg = command.split('_', maxsplit=1) - if command == 'savefile': - yt_id = self._savefile(arg) + if command == 'abort': + yt_id = self._abort_downloading() else: - yt_id = arg - if command == 'forget': - self._forget_file(yt_id) - elif command == 'queue': - self._queue_download(yt_id) + command, arg = command.split('_', maxsplit=1) + if command == 'savefile': + yt_id = self._savefile(arg) + elif command in {'unqueue', 'up', 'down'}: + idx = int(arg) + if command == 'unqueue': + yt_id = self._unqueue_download(idx) + else: + self._move_in_queue(idx, upwards=command == 'up') + continue + else: + yt_id = arg + if command == 'queue': + self._queue_download(yt_id) + elif command == 'forget': + self._forget_file(yt_id) self._update_timestamp(yt_id) Thread(target=loop, daemon=False).start()