Add table schema validation.

diff --git a/src/migrations/0_init.sql b/src/migrations/0_init.sql
index 5b45769cf958d7f223d224412a9f5bae8a73ded7..24c4a3f6530a658052c7328e9a768876f342292e 100644 (file)
--- a/src/migrations/0_init.sql
+++ b/src/migrations/0_init.sql
@@ -1,9 +1,27 @@
-CREATE TABLE yt_queries (
+CREATE TABLE "files" (
+  rel_path TEXT PRIMARY KEY,
+  yt_id TEXT NOT NULL DEFAULT "",
+  flags INTEGER NOT NULL DEFAULT 0,
+  FOREIGN KEY (yt_id) REFERENCES "yt_videos"(id)
+);
+CREATE TABLE "quota_costs" (
+  id TEXT PRIMARY KEY,
+  timestamp TEXT NOT NULL,
+  cost INT NOT NULL
+);
+CREATE TABLE "yt_queries" (

   id TEXT PRIMARY KEY,
   text TEXT NOT NULL,
   retrieved_at TEXT NOT NULL
 );
   id TEXT PRIMARY KEY,
   text TEXT NOT NULL,
   retrieved_at TEXT NOT NULL
 );

-CREATE TABLE yt_videos (
+CREATE TABLE "yt_query_results" (
+  query_id TEXT NOT NULL,
+  video_id TEXT NOT NULL,
+  PRIMARY KEY (query_id, video_id),
+  FOREIGN KEY (query_id) REFERENCES "yt_queries"(id),
+  FOREIGN KEY (video_id) REFERENCES "yt_videos"(id)
+);
+CREATE TABLE "yt_videos" (

   id TEXT PRIMARY KEY,
   title TEXT NOT NULL,
   description TEXT NOT NULL,
   id TEXT PRIMARY KEY,
   title TEXT NOT NULL,
   description TEXT NOT NULL,


@@ -11,22 +29,3 @@ CREATE TABLE yt_videos (
   duration TEXT NOT NULL,
   definition TEXT NOT NULL
 );
   duration TEXT NOT NULL,
   definition TEXT NOT NULL
 );

-CREATE TABLE yt_query_results (
-  query_id TEXT NOT NULL,
-  video_id TEXT NOT NULL,
-  PRIMARY KEY (query_id, video_id),
-  FOREIGN KEY (query_id) REFERENCES yt_queries(id),
-  FOREIGN KEY (video_id) REFERENCES yt_videos(id)
-);
-CREATE TABLE quota_costs (
-  id TEXT PRIMARY KEY,
-  timestamp TEXT NOT NULL,
-  cost INT NOT NULL
-);
-CREATE TABLE files (
-  rel_path TEXT PRIMARY KEY,
-  yt_id TEXT NOT NULL DEFAULT "",
-  flags INTEGER NOT NULL DEFAULT 0,
-  FOREIGN KEY (yt_id) REFERENCES yt_videos(id)
-);
-

diff --git a/src/migrations/1_add_files_last_updated.sql b/src/migrations/1_add_files_last_updated.sql
index 678d84322196bdfe726e1d0e59f387fffc00eaa5..4d2427e464fe784229b8c3253ee0247089c00104 100644 (file)
--- a/src/migrations/1_add_files_last_updated.sql
+++ b/src/migrations/1_add_files_last_updated.sql
@@ -1 +1 @@
-ALTER TABLE files ADD COLUMN last_update TEXT NOT NULL DEFAULT "2000-01-01 12:00:00.123456";
+ALTER TABLE "files" ADD COLUMN last_update TEXT NOT NULL DEFAULT "2000-01-01 12:00:00.123456";

diff --git a/src/migrations/2_add_files_sha512.sql b/src/migrations/2_add_files_sha512.sql
index 36d99e11e28c171bfd0d2a0a4a83882d5fb4939e..33f08d8b7c4256ed364e00fba917915ee592670a 100644 (file)
--- a/src/migrations/2_add_files_sha512.sql
+++ b/src/migrations/2_add_files_sha512.sql
@@ -1 +1 @@
-ALTER TABLE files ADD COLUMN sha512_digest TEXT NOT NULL DEFAULT "";
+ALTER TABLE "files" ADD COLUMN sha512_digest TEXT NOT NULL DEFAULT "";

diff --git a/src/migrations/3_files_redo.sql b/src/migrations/3_files_redo.sql
index bf485bb1036f613dab80ec2caf6cdd908e0dccce..9c0ff8780181ba6b7ceed9fc1ccbcd875e685f2c 100644 (file)
--- a/src/migrations/3_files_redo.sql
+++ b/src/migrations/3_files_redo.sql
@@ -1,32 +1,32 @@
-CREATE TEMPORARY TABLE files_backup (
+CREATE TEMPORARY TABLE "files_backup" (

   rel_path TEXT PRIMARY KEY,
   yt_id TEXT NOT NULL DEFAULT "",
   flags INTEGER NOT NULL DEFAULT 0,
   last_update TEXT NOT NULL DEFAULT "2000-01-01 12:00:00.123456",
   sha512_digest TEXT NOT NULL,
   rel_path TEXT PRIMARY KEY,
   yt_id TEXT NOT NULL DEFAULT "",
   flags INTEGER NOT NULL DEFAULT 0,
   last_update TEXT NOT NULL DEFAULT "2000-01-01 12:00:00.123456",
   sha512_digest TEXT NOT NULL,

-  FOREIGN KEY (yt_id) REFERENCES yt_videos(id)
+  FOREIGN KEY (yt_id) REFERENCES "yt_videos"(id)

 );
 );

-INSERT INTO files_backup SELECT
+INSERT INTO "files_backup" SELECT

   rel_path,
   yt_id,
   flags,
   last_update,
   sha512_digest
   rel_path,
   yt_id,
   flags,
   last_update,
   sha512_digest

-FROM files;
-DROP TABLE files;
-CREATE TABLE files (
+FROM "files";
+DROP TABLE "files";
+CREATE TABLE "files" (

   sha512_digest TEXT PRIMARY KEY,
   rel_path TEXT NOT NULL,
   flags INTEGER NOT NULL DEFAULT 0,
   yt_id TEXT,
   last_update TEXT NOT NULL,
   sha512_digest TEXT PRIMARY KEY,
   rel_path TEXT NOT NULL,
   flags INTEGER NOT NULL DEFAULT 0,
   yt_id TEXT,
   last_update TEXT NOT NULL,

-  FOREIGN KEY (yt_id) REFERENCES yt_videos(id)
+  FOREIGN KEY (yt_id) REFERENCES "yt_videos"(id)

 );
 );

-INSERT INTO files SELECT
+INSERT INTO "files" SELECT

   sha512_digest,
   rel_path,
   flags,
   yt_id,
   last_update
   sha512_digest,
   rel_path,
   flags,
   yt_id,
   last_update

-FROM files_backup;
-DROP TABLE files_backup;
+FROM "files_backup";
+DROP TABLE "files_backup";

diff --git a/src/migrations/4_add_files_sha512_blob.sql b/src/migrations/4_add_files_sha512_blob.sql
index c3827407b1628be74040daad878ba12caab8c616..4ed2299c7337c8151503ffc5746b97b1e9667ba5 100644 (file)
--- a/src/migrations/4_add_files_sha512_blob.sql
+++ b/src/migrations/4_add_files_sha512_blob.sql
@@ -1 +1 @@
-ALTER TABLE files ADD COLUMN sha512_blob BLOB;
+ALTER TABLE "files" ADD COLUMN sha512_blob BLOB;

diff --git a/src/migrations/5_files_redo.sql b/src/migrations/5_files_redo.sql
index a9c30fcb473bee2296f9a7439209227d7f278d1c..44bbb92a8721c42009c8fab5ae87b96ff0b1be9d 100644 (file)
--- a/src/migrations/5_files_redo.sql
+++ b/src/migrations/5_files_redo.sql
@@ -1,17 +1,17 @@
-CREATE TABLE files_new (
+ALTER TABLE "files" RENAME TO "files_old";
+CREATE TABLE "files" (

   digest BLOB PRIMARY KEY,
   rel_path TEXT NOT NULL,
   flags INTEGER NOT NULL DEFAULT 0,
   yt_id TEXT,
   last_update TEXT NOT NULL,
   digest BLOB PRIMARY KEY,
   rel_path TEXT NOT NULL,
   flags INTEGER NOT NULL DEFAULT 0,
   yt_id TEXT,
   last_update TEXT NOT NULL,

-  FOREIGN KEY (yt_id) REFERENCES yt_videos(id)
+  FOREIGN KEY (yt_id) REFERENCES "yt_videos"(id)

 );
 );

-INSERT INTO files_new SELECT
+INSERT INTO "files" SELECT

   sha512_blob,
   rel_path,
   flags,
   yt_id,
   last_update
   sha512_blob,
   rel_path,
   flags,
   yt_id,
   last_update

-FROM files;
-DROP TABLE files;
-ALTER TABLE files_new RENAME TO files;
+FROM "files_old";
+DROP TABLE "files_old";

diff --git a/src/migrations/6_add_files_tags.sql b/src/migrations/6_add_files_tags.sql
index 7d70c183f3310bc87d7120a591f173134b4534d5..407783020a17860ff071f02b7c8c3a8008a6c9a2 100644 (file)
--- a/src/migrations/6_add_files_tags.sql
+++ b/src/migrations/6_add_files_tags.sql
@@ -1 +1 @@
-ALTER TABLE files ADD COLUMN tags TEXT NOT NULL DEFAULT "";
+ALTER TABLE "files" ADD COLUMN tags TEXT NOT NULL DEFAULT "";

diff --git a/src/migrations/new_init.sql b/src/migrations/new_init.sql
index ad116702752b33c976fa9d7380cd93dad19e2b7b..ae5e871dfc63c861e434920ca50908a177ea1f51 100644 (file)
--- a/src/migrations/new_init.sql
+++ b/src/migrations/new_init.sql
@@ -1,9 +1,30 @@
-CREATE TABLE yt_queries (
+CREATE TABLE "files" (
+  digest BLOB PRIMARY KEY,
+  rel_path TEXT NOT NULL,
+  flags INTEGER NOT NULL DEFAULT 0,
+  yt_id TEXT,
+  last_update TEXT NOT NULL,
+  tags TEXT NOT NULL DEFAULT "",
+  FOREIGN KEY (yt_id) REFERENCES "yt_videos"(id)
+);
+CREATE TABLE "quota_costs" (
+  id TEXT PRIMARY KEY,
+  timestamp TEXT NOT NULL,
+  cost INT NOT NULL
+);
+CREATE TABLE "yt_queries" (

   id TEXT PRIMARY KEY,
   text TEXT NOT NULL,
   retrieved_at TEXT NOT NULL
 );
   id TEXT PRIMARY KEY,
   text TEXT NOT NULL,
   retrieved_at TEXT NOT NULL
 );

-CREATE TABLE yt_videos (
+CREATE TABLE "yt_query_results" (
+  query_id TEXT NOT NULL,
+  video_id TEXT NOT NULL,
+  PRIMARY KEY (query_id, video_id),
+  FOREIGN KEY (query_id) REFERENCES "yt_queries"(id),
+  FOREIGN KEY (video_id) REFERENCES "yt_videos"(id)
+);
+CREATE TABLE "yt_videos" (

   id TEXT PRIMARY KEY,
   title TEXT NOT NULL,
   description TEXT NOT NULL,
   id TEXT PRIMARY KEY,
   title TEXT NOT NULL,
   description TEXT NOT NULL,


@@ -11,24 +32,3 @@ CREATE TABLE yt_videos (
   duration TEXT NOT NULL,
   definition TEXT NOT NULL
 );
   duration TEXT NOT NULL,
   definition TEXT NOT NULL
 );

-CREATE TABLE yt_query_results (
-  query_id TEXT NOT NULL,
-  video_id TEXT NOT NULL,
-  PRIMARY KEY (query_id, video_id),
-  FOREIGN KEY (query_id) REFERENCES yt_queries(id),
-  FOREIGN KEY (video_id) REFERENCES yt_videos(id)
-);
-CREATE TABLE quota_costs (
-  id TEXT PRIMARY KEY,
-  timestamp TEXT NOT NULL,
-  cost INT NOT NULL
-);
-CREATE TABLE files (
-  digest BLOB PRIMARY KEY,
-  rel_path TEXT NOT NULL,
-  flags INTEGER NOT NULL DEFAULT 0,
-  yt_id TEXT,
-  last_update TEXT NOT NULL,
-  tags TEXT NOT NULL DEFAULT "",
-  FOREIGN KEY (yt_id) REFERENCES yt_videos(id)
-);

diff --git a/src/ytplom/db.py b/src/ytplom/db.py
index e30e2ad0181bc4164b692341540ff147ca079bdc..51e94ac064c9c98c7b8209542c46b7c7ceaa57a6 100644 (file)
--- a/src/ytplom/db.py
+++ b/src/ytplom/db.py
@@ -2,6 +2,7 @@
 
 # included libs
 from base64 import urlsafe_b64decode, urlsafe_b64encode
 
 # included libs
 from base64 import urlsafe_b64decode, urlsafe_b64encode

+from difflib import Differ

 from hashlib import file_digest
 from pathlib import Path
 from sqlite3 import (connect as sql_connect, Connection as SqlConnection,
 from hashlib import file_digest
 from pathlib import Path
 from sqlite3 import (connect as sql_connect, Connection as SqlConnection,


@@ -63,16 +64,63 @@ class DbFile:
 
     def __init__(self,
                  path: Path = PATH_DB,
 
     def __init__(self,
                  path: Path = PATH_DB,

-                 expected_version: int = EXPECTED_DB_VERSION
+                 version_to_validate: int = EXPECTED_DB_VERSION

                  ) -> None:
         self._path = path
                  ) -> None:
         self._path = path

-        if not path.is_file():
-            raise HandledException(f'no DB file at {path}')
-        if expected_version >= 0:
-            user_version = self._get_user_version()
-            if user_version != expected_version:
-                raise HandledException(
-                    f'wrong DB version {user_version} (!= {expected_version})')
+        if not self._path.is_file():
+            raise HandledException(f'no DB file at {self._path}')
+
+        if version_to_validate < 0:
+            return
+        # ensure version
+        if (user_version := self._get_user_version()) != version_to_validate:
+            raise HandledException(
+                f'wrong DB version {user_version} (!= {version_to_validate})')
+
+        # ensure schema
+        with sql_connect(self._path) as conn:
+            schema_rows = [
+                    r[0] for r in
+                    conn.execute('SELECT sql FROM sqlite_master ORDER BY sql')
+                    if r[0]]
+        schema_rows_normed = []
+        indent = '  '
+        for row in schema_rows:
+            row_normed = []
+            for subrow in row.split('\n'):
+                subrow = subrow.rstrip()
+                in_parentheses = 0
+                split_at = []
+                for i, c in enumerate(subrow):
+                    if '(' == c:
+                        in_parentheses += 1
+                    elif ')' == c:
+                        in_parentheses -= 1
+                    elif ',' == c and 0 == in_parentheses:
+                        split_at += [i + 1]
+                prev_split = 0
+                for i in split_at:
+                    segment = subrow[prev_split:i].strip()
+                    if len(segment) > 0:
+                        row_normed += [f'{indent}{segment}']
+                    prev_split = i
+                segment = subrow[prev_split:].strip()
+                if len(segment) > 0:
+                    row_normed += [f'{indent}{segment}']
+            row_normed[0] = row_normed[0].lstrip()
+            row_normed[-1] = row_normed[-1].lstrip()
+            if row_normed[-1] != ')' and row_normed[-3][-1] != ',':
+                row_normed[-3] = row_normed[-3] + ','
+                row_normed[-2:] = [indent + row_normed[-1][:-1]] + [')']
+            schema_rows_normed += ['\n'.join(row_normed)]
+        retrieved_schema = ';\n'.join(schema_rows_normed) + ';'
+        with open(_PATH_DB_SCHEMA, 'r', encoding='utf-8') as f:
+            stored_schema = f.read().rstrip()
+        if stored_schema != retrieved_schema:
+            diff_msg = Differ().compare(retrieved_schema.splitlines(),
+                                        stored_schema.splitlines())
+            raise HandledException('DB has wrong tables schema. Diff:\n'
+                                   + '\n'.join(diff_msg))

 
     def _get_user_version(self) -> int:
         with sql_connect(self._path) as conn:
 
     def _get_user_version(self) -> int:
         with sql_connect(self._path) as conn:

diff --git a/src/ytplom/migrations.py b/src/ytplom/migrations.py
index cb038ef598e9a588df320dd712447d753fb9ec78..5cacc9588790831fab2d79c785cb8241f6342b21 100644 (file)
--- a/src/ytplom/migrations.py
+++ b/src/ytplom/migrations.py
@@ -68,4 +68,4 @@ _MIGRATIONS: MigrationsList = [
 
 def migrate():
     """Migrate DB file at expected default path to most recent version."""
 
 def migrate():
     """Migrate DB file at expected default path to most recent version."""

-    DbFile(expected_version=-1).migrate(_MIGRATIONS)
+    DbFile(version_to_validate=-1).migrate(_MIGRATIONS)