Implemented element_lookup_cache

- not used for now

server/app.py

@@ -46,6 +46,7 @@ from entertainment_decider.models import (
     MediaElement,
     generate_preference_list,
     setup_custom_tables,
+    update_element_lookup_cache,
 )
 from entertainment_decider.extractors.collection import (
     collection_extract_uri,
@@ -417,6 +418,7 @@ def refresh_collections():
         orm.select(c.id for c in MediaCollection if c.keep_updated)
     )
     errors = []
+    failed_colls = set[int]()
     for coll_id in collection_ids:
         try:
             coll = MediaCollection[coll_id]
@@ -425,6 +427,7 @@ def refresh_collections():
         # TODO make Exception more specific
         except Exception as e:
             orm.rollback()
+            failed_colls.add(coll_id)
             coll = MediaCollection[coll_id]
             errors.append(
                 {
@@ -438,6 +441,9 @@ def refresh_collections():
                     },
                 },
             )
+    # TODO detect changed collections properly to speed up cache rebuild
+    # meaning check if collection really changed
+    update_element_lookup_cache(collection_ids - failed_colls)
     if errors:
         return (
             {
@@ -563,6 +569,8 @@ def api_collection_extract():
         }
     c = collection_extract_uri(data["uri"])
     orm.flush()
+    if c:
+        update_element_lookup_cache([c.id])
     if c and environ_bool(data.get("redirect_to_object", False)):
         return redirect(c.info_link)
     return redirect_back_or_okay()
@@ -618,6 +626,7 @@ def api_collection_element(collection_id: int):
         collection.set(**{key: KEY_CONVERTER[key](val) for key, val in data.items()})
         if "watch_in_order" in data:  # TODO move both to property inside class
             collection.watch_in_order_auto = False
+            update_element_lookup_cache([collection.id])
         return redirect_back_or_okay()
     else:
         return {

server/entertainment_decider/models.py

@@ -8,6 +8,7 @@ import gzip
 import json
 import math
 import logging
+import re
 from typing import (
     Callable,
     Dict,
@@ -702,6 +703,65 @@ class CollectionUriMapping(db.Entity):
     element: MediaCollection = orm.Required(MediaCollection)
 
 
+SQL_WHITESPACE_PATTERN = re.compile(r"(\s|\n)+")
+
+
+def sql_cleanup(sql: str) -> str:
+    return SQL_WHITESPACE_PATTERN.sub(" ", sql).strip()
+
+
+def update_element_lookup_cache(collection_ids: List[int] = []):
+    logging.info(
+        f"Rebuild element_lookup_cache for {len(collection_ids) if collection_ids else 'all'} collections …"
+    )
+
+    def filter_clause(c_id: str):
+        return sql_where_in(c_id, collection_ids) if collection_ids else "true"
+
+    orm.flush()
+    sql = [
+        f"""
+            DELETE QUICK FROM element_lookup_cache
+            WHERE {filter_clause("collection")};
+        """,
+        f"""
+            INSERT INTO element_lookup_cache (collection, element1, element2) SELECT
+                c.id AS collection,
+                l1.element AS element1,
+                l2.element AS element2
+            FROM
+                mediacollection c
+            INNER JOIN mediacollectionlink l1 ON
+                c.id = l1.collection
+            INNER JOIN mediacollectionlink l2 ON
+                c.id = l2.collection
+            INNER JOIN mediaelement e1 ON
+                l1.element = e1.id
+            INNER JOIN mediaelement e2 ON
+                l2.element = e2.id
+            WHERE
+                (
+                    l1.season,
+                    l1.episode,
+                    e1.release_date,
+                    e1.id
+                ) <(
+                    l2.season,
+                    l2.episode,
+                    e2.release_date,
+                    e2.id
+                ) AND c.watch_in_order
+                AND {filter_clause("c.id")}
+            GROUP BY
+                collection,
+                element1,
+                element2
+        """,
+    ]
+    for q in sql:
+        db.execute(sql_cleanup(q))
+
+
 ####
 ## Custom Table Framework
 ####
@@ -711,6 +771,46 @@ class CollectionUriMapping(db.Entity):
 CUSTOM_TABLE_DEFINITIONS: Mapping[SafeStr, str] = {
     SafeStr(table_name): trim(table_sql)
     for table_name, table_sql in {
+        "element_lookup_cache": """
+            CREATE TABLE element_lookup_cache(
+                collection INT(11) NOT NULL,
+                element1 INT(11) NOT NULL,
+                element2 INT(11) NOT NULL
+            ) SELECT
+                c.id AS collection,
+                l1.element AS element1,
+                l2.element AS element2
+            FROM
+                mediacollection c
+            INNER JOIN mediacollectionlink l1 ON
+                c.id = l1.collection
+            INNER JOIN mediacollectionlink l2 ON
+                c.id = l2.collection
+            INNER JOIN mediaelement e1 ON
+                l1.element = e1.id
+            INNER JOIN mediaelement e2 ON
+                l2.element = e2.id
+            WHERE
+                (
+                    l1.season,
+                    l1.episode,
+                    e1.release_date,
+                    e1.id
+                ) <(
+                    l2.season,
+                    l2.episode,
+                    e2.release_date,
+                    e2.id
+                ) AND c.watch_in_order
+            GROUP BY
+                collection,
+                element1,
+                element2;
+            ALTER TABLE
+                `element_lookup_cache` ADD PRIMARY KEY(`element1`, `element2`, `collection`);
+            ALTER TABLE
+                `element_lookup_cache` ADD INDEX(`collection`);
+        """,
     }
 }