server: improve migration, camera now UnicodeText

- [server/migration] merge add_camera -> add_date_taken
- [server/migration] rename add_date_taken -> add_date_taken_and_camera
- [server/migration] add_date_taken_and_camera: iterate through posts in db
  instead of files
- [server] changed Post.camera from sa.Text -> sa.UnicodeText
- [server] improved stability of `func/images.py`
- [todo] will add migration that fixes image dimensions and regenerates
  thumbnails later

server/szurubooru/func/images.py

@@ -172,7 +172,7 @@ class Image:
                     camera_string.append(tags[option])
 
             if camera_string:
-                self.camera_string = " ".join(camera_string)
+                self.camera = " ".join(camera_string)
         else:
             raise Exception
 
@@ -414,7 +414,8 @@ class Video:
         assert "format" in info
         assert "tags" in info["format"]
 
-        self.date_taken = info["format"]["tags"]["creation_time"]
+        if "creation_time" in info["format"]["tags"]:
+            self.date_taken = info["format"]["tags"]["creation_time"]
 
         # List of tuples where only one value can be valid
         option_tuples = (

server/szurubooru/migrations/versions/57aafb3bf9f0_add_date_taken.py

@@ -1,24 +0,0 @@
-"""
-add_date_taken
-
-Revision ID: 57aafb3bf9f0
-Created at: 2021-11-26 21:00:19.698012
-"""
-
-import sqlalchemy as sa
-from alembic import op
-
-revision = "57aafb3bf9f0"
-down_revision = "adcd63ff76a2"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column(
-        "post", sa.Column("date_taken", sa.DateTime(), nullable=True)
-    )
-
-
-def downgrade():
-    op.drop_column("post", "date_taken")

server/szurubooru/migrations/versions/57aafb3bf9f0_add_date_taken_and_camera.py

@@ -0,0 +1,64 @@
+"""
+Add camera and date_taken, and fill them out
+
+Revision ID: 57aafb3bf9f0
+Created at: 2022-01-07 17:54:15.982212
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+from szurubooru.func.files import _get_full_path
+from szurubooru.func.images import Image, Video
+from szurubooru.func.mime import get_extension
+from szurubooru.func.posts import get_post_security_hash
+
+revision = "57aafb3bf9f0"
+down_revision = "adcd63ff76a2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    conn = op.get_bind()
+
+    op.add_column("post", sa.Column("camera", sa.Text(), nullable=True))
+
+    op.add_column(
+        "post", sa.Column("date_taken", sa.DateTime(), nullable=True)
+    )
+
+    posts = sa.Table(
+        "post",
+        sa.MetaData(),
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("camera", sa.Text, nullable=True),
+        sa.Column("date_taken", sa.DateTime, nullable=True),
+        sa.Column("mime-type", sa.Unicode(32), nullable=False),
+        sa.Column("type", sa.Unicode(32), nullable=False),
+    )
+
+    for post in conn.execute(posts.select().where(posts.c.type != "flash")):
+        ext = get_extension(post["mime-type"])
+        filename = f"{post.id}_{get_post_security_hash(post.id)}.{ext}"
+
+        content = open(_get_full_path("posts/" + filename), "rb").read()
+
+        if post.type == "image":
+            media = Image(content)
+        else:
+            media = Video(content)
+
+        conn.execute(
+            posts.update()
+            .where(posts.c.id == post.id)
+            .values(
+                camera=media.camera,
+                date_taken=media.date_taken,
+            )
+        )
+
+
+def downgrade():
+    op.drop_column("post", "camera")
+    op.drop_column("post", "date_taken")

server/szurubooru/migrations/versions/adb2acef2492_add_camera.py

@@ -1,59 +0,0 @@
-"""
-add_camera
-
-Revision ID: adb2acef2492
-Created at: 2021-12-01 13:06:14.285699
-"""
-
-import sqlalchemy as sa
-from alembic import op
-
-from szurubooru.func import files, metadata
-
-revision = "adb2acef2492"
-down_revision = "57aafb3bf9f0"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    conn = op.get_bind()
-
-    op.add_column("post", sa.Column("camera", sa.Text(), nullable=True))
-
-    posts = sa.Table(
-        "post",
-        sa.MetaData(),
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column("camera", sa.Text, nullable=True),
-    )
-
-    for file in list(files.scan("posts")):
-        filename = file.name
-        fullpath = files._get_full_path("posts/" + filename)
-
-        post_ext = filename.split(".")[1]
-
-        if post_ext in ["jpg", "jpeg", "png", "heif", "heic"]:
-            with open(fullpath, "rb") as img:
-                camera_string = metadata.resolve_image_camera(img)
-        elif post_ext in ["webm", "mp4", "avif"]:
-            camera_string = metadata.resolve_video_camera(
-                files._get_full_path(fullpath)
-            )
-        else:
-            continue
-
-        post_id = int(filename.split("_")[0])
-
-        conn.execute(
-            posts.update()
-            .where(posts.c.id == post_id)
-            .values(camera=camera_string)
-        )
-
-    op.alter_column("post", "camera", nullable=True)
-
-
-def downgrade():
-    op.drop_column("post", "camera")

server/szurubooru/model/post.py

@@ -225,7 +225,7 @@ class Post(Base):
 
     # EXIF things
     date_taken = sa.Column("date_taken", sa.DateTime, nullable=True)
-    camera = sa.Column("camera", sa.Text, nullable=True)
+    camera = sa.Column("camera", sa.UnicodeText, nullable=True)
 
     # foreign tables
     user = sa.orm.relationship("User")