From: Christian Heller Date: Fri, 15 Nov 2024 04:06:37 +0000 (+0100) Subject: Move quota logging into sqlite DB, too. X-Git-Url: https://plomlompom.com/repos/%7B%7B%20web_path%20%7D%7D/%7B%7Bprefix%7D%7D/%7B%7Bdb.prefix%7D%7D/edit?a=commitdiff_plain;h=08214286970742abf81c21df55fdeb98eeb193c0;p=ytplom Move quota logging into sqlite DB, too. --- diff --git a/ytplom.py b/ytplom.py index cfabedc..abb755b 100755 --- a/ytplom.py +++ b/ytplom.py @@ -6,7 +6,8 @@ from os.path import (isdir, isfile, exists as path_exists, join as path_join, splitext, basename) from random import shuffle from time import time, sleep -from json import load as json_load, dump as json_dump, dumps as json_dumps +from json import dumps as json_dumps +from uuid import uuid4 from datetime import datetime, timedelta from threading import Thread from http.server import HTTPServer, BaseHTTPRequestHandler @@ -43,7 +44,6 @@ class NotFoundException(BaseException): """Call on DB fetches finding less than expected.""" -PATH_QUOTA_LOG = PathStr('quota_log.json') PATH_DIR_DOWNLOADS = PathStr('downloads') PATH_DIR_THUMBNAILS = PathStr('thumbnails') PATH_DIR_REQUESTS_CACHE = PathStr('cache_googleapi') @@ -95,6 +95,11 @@ CREATE TABLE yt_query_results ( FOREIGN KEY (query_id) REFERENCES yt_queries(id), FOREIGN KEY (video_id) REFERENCES yt_videos(id) ); +CREATE TABLE quota_costs ( + id TEXT PRIMARY KEY, + timestamp TEXT NOT NULL, + cost INT NOT NULL +); ''' to_download: list[VideoId] = [] @@ -255,6 +260,43 @@ class VideoData(DbData): (query_id, self.id_)) +class DbQuotaCost(DbData): + """Collects API access quota costs.""" + _table_name = 'quota_costs' + _cols = ('id_', 'timestamp', 'cost') + + def __init__(self, + id_: Optional[str], + timestamp: DatetimeStr, + cost: QuotaCost + ) -> None: + self.id_ = id_ if id_ else str(uuid4()) + self.timestamp = timestamp + self.cost = cost + + @classmethod + def update(cls, conn: DatabaseConnection, cost: QuotaCost) -> None: + """Adds cost mapped to current datetime.""" + cls._remove_old(conn) + new = cls(None, + DatetimeStr(datetime.now().strftime(TIMESTAMP_FMT)), + QuotaCost(cost)) + new.save(conn) + + @classmethod + def current(cls, conn: DatabaseConnection) -> QuotaCost: + """Returns quota cost total for last 24 hours, purges old data.""" + cls._remove_old(conn) + quota_costs = cls.get_all(conn) + return QuotaCost(sum(c.cost for c in quota_costs)) + + @classmethod + def _remove_old(cls, conn: DatabaseConnection) -> None: + cutoff = datetime.now() - timedelta(days=1) + sql = SqlText(f'DELETE FROM {cls._table_name} WHERE timestamp < ?') + conn.exec(SqlText(sql), (cutoff.strftime(TIMESTAMP_FMT),)) + + class Player: """MPV representation with some additional features.""" @@ -386,15 +428,6 @@ def ensure_expected_dirs_and_files() -> None: elif not isdir(dir_name): msg = f'at expected directory path {dir_name} found non-directory' raise Exception(msg) - if not path_exists(PATH_QUOTA_LOG): - with open(PATH_QUOTA_LOG, 'w', encoding='utf8') as f: - f.write('{}') - else: - try: - read_quota_log() # just to check if we can - except Exception as e: - print(f'Trouble reading quota log file at {PATH_QUOTA_LOG}:') - raise e def clean_unfinished_downloads() -> None: @@ -416,27 +449,6 @@ def run_server() -> None: server.server_close() -def read_quota_log() -> QuotaLog: - """Return logged quota expenditures of past 24 hours.""" - with open(PATH_QUOTA_LOG, 'r', encoding='utf8') as f: - log = json_load(f) - ret = {} - now = datetime.now() - for timestamp, amount in log.items(): - then = datetime.strptime(timestamp, TIMESTAMP_FMT) - if then >= now - timedelta(days=1): - ret[timestamp] = amount - return ret - - -def update_quota_log(now: DatetimeStr, cost: QuotaCost) -> None: - """Update quota log from read_quota_log, add cost to now's row.""" - quota_log = read_quota_log() - quota_log[now] = QuotaCost(quota_log.get(now, 0) + cost) - with open(PATH_QUOTA_LOG, 'w', encoding='utf8') as f: - json_dump(quota_log, f) - - def download_thread() -> None: """Keep iterating through to_download for IDs, download their videos.""" while True: @@ -491,11 +503,12 @@ class TaskHandler(BaseHTTPRequestHandler): self._send_http(headers=[('Location', '/')], code=302) def _post_query(self, query_txt: QueryText) -> None: + conn = DatabaseConnection() - def collect_results(now, query_txt: QueryText) -> list[VideoData]: + def collect_results(query_txt: QueryText) -> list[VideoData]: youtube = googleapiclient.discovery.build('youtube', 'v3', developerKey=API_KEY) - update_quota_log(now, QUOTA_COST_YOUTUBE_SEARCH) + DbQuotaCost.update(conn, QUOTA_COST_YOUTUBE_SEARCH) search_request = youtube.search().list( q=query_txt, part='snippet', @@ -514,7 +527,7 @@ class TaskHandler(BaseHTTPRequestHandler): title=snippet['title'], description=snippet['description'], published_at=snippet['publishedAt'])] - update_quota_log(now, QUOTA_COST_YOUTUBE_DETAILS) + DbQuotaCost.update(conn, QUOTA_COST_YOUTUBE_DETAILS) ids_for_details = ','.join([r.id_ for r in results]) videos_request = youtube.videos().list(id=ids_for_details, part='content_details') @@ -526,11 +539,11 @@ class TaskHandler(BaseHTTPRequestHandler): result.definition = content_details['definition'].upper() return results - now = DatetimeStr(datetime.now().strftime(TIMESTAMP_FMT)) - query_data = QueryData(None, query_txt, now) - conn = DatabaseConnection() + query_data = QueryData( + None, query_txt, + DatetimeStr(datetime.now().strftime(TIMESTAMP_FMT))) query_data.save(conn) - for result in collect_results(now, query_txt): + for result in collect_results(query_txt): result.save(conn) assert query_data.id_ is not None result.save_to_query(conn, query_data.id_) @@ -611,8 +624,8 @@ class TaskHandler(BaseHTTPRequestHandler): {'query': query.text, 'videos': results}) def _send_queries_index_and_search(self) -> None: - quota_count = QuotaCost(sum(read_quota_log().values())) conn = DatabaseConnection() + quota_count = DbQuotaCost.current(conn) queries_data = QueryData.get_all(conn) conn.commit_close() queries_data.sort(key=lambda q: q.retrieved_at, reverse=True)