server/tests: update unit tests for elasticsearch removal

server/szurubooru/tests/api/test_post_creating.py

@@ -12,6 +12,7 @@ def inject_config(config_injector):
             'posts:create:identified': model.User.RANK_REGULAR,
             'tags:create': model.User.RANK_REGULAR,
         },
+        'allow_broken_uploads': False,
     })
 
 
@@ -250,8 +251,7 @@ def test_omitting_optional_field(
 
 
 def test_errors_not_spending_ids(
-        config_injector, tmpdir, context_factory, read_asset, user_factory,
+        config_injector, tmpdir, context_factory, read_asset, user_factory):
-        skip_post_hashing):
     config_injector({
         'data_dir': str(tmpdir.mkdir('data')),
         'data_url': 'example.com',

server/szurubooru/tests/api/test_post_updating.py

@@ -19,6 +19,7 @@ def inject_config(config_injector):
             'posts:edit:thumbnail': model.User.RANK_REGULAR,
             'tags:create': model.User.RANK_MODERATOR,
         },
+        'allow_broken_uploads': False,
     })
 
 

server/szurubooru/tests/conftest.py

@@ -139,15 +139,8 @@ def tag_factory():
     return factory
 
 
-@pytest.yield_fixture
-def skip_post_hashing():
-    with patch('szurubooru.func.image_hash.add_image'), \
-            patch('szurubooru.func.image_hash.delete_image'):
-        yield
-
-
 @pytest.fixture
-def post_factory(skip_post_hashing):
+def post_factory():
     # pylint: disable=invalid-name
     def factory(
             id=None,

server/szurubooru/tests/func/test_image_hash.py

@@ -1,37 +1,26 @@
 import pytest
 from szurubooru.func import image_hash
+from numpy import array_equal
 
 
-def test_hashing(read_asset, config_injector):
+def test_signature_functions(read_asset, config_injector):
-    config_injector({
+    sig1 = image_hash.generate_signature(read_asset('jpeg.jpg'))
-        'elasticsearch': {
+    sig2 = image_hash.generate_signature(read_asset('jpeg-similar.jpg'))
-            'host': 'localhost',
-            'port': 9200,
-            'index': 'szurubooru_test',
-            'user': 'szurubooru',
-            'pass': None,
-        },
-    })
 
-    if not image_hash.get_session().ping():
+    sig1_repacked = image_hash.unpack_signature(
-        pytest.xfail(
+        image_hash.pack_signature(sig1))
-            'Unable to connect to ElasticSearch, '
+    sig2_repacked = image_hash.unpack_signature(
-            'perhaps it is not available for this test?')
+        image_hash.pack_signature(sig2))
+    assert array_equal(sig1, sig1_repacked)
+    assert array_equal(sig2, sig2_repacked)
 
-    image_hash.purge()
+    dist1 = image_hash.normalized_distance([sig1], sig2)
-    image_hash.add_image('test', read_asset('jpeg.jpg'))
+    assert abs(dist1[0] - 0.20599895341812172) < 1e-8
 
-    paths = image_hash.get_all_paths()
+    dist2 = image_hash.normalized_distance([sig2], sig2)
-    results_exact = image_hash.search_by_image(read_asset('jpeg.jpg'))
+    assert abs(dist2[0]) < 1e-8
-    results_similar = image_hash.search_by_image(
-        read_asset('jpeg-similar.jpg'))
 
-    assert len(paths) == 1
+    words1 = image_hash.generate_words(sig1)
-    assert len(results_exact) == 1
+    words2 = image_hash.generate_words(sig2)
-    assert len(results_similar) == 1
+    words_match = sum(word1 == word2 for word1, word2 in zip(words1, words2))
-    assert results_exact[0].path == 'test'
+    assert words_match == 17
-    assert results_exact[0].score == 63
-    assert results_exact[0].distance == 0
-    assert results_similar[0].path == 'test'
-    assert results_similar[0].score == 17
-    assert abs(results_similar[0].distance - 0.20599895341812172) < 1e-8

server/szurubooru/tests/func/test_posts.py

@@ -354,6 +354,7 @@ def test_update_post_content_for_new_post(
                 'post_height': 300,
             },
             'secret': 'test',
+            'allow_broken_uploads': False,
         })
         output_file_path = '{}/data/posts/{}'.format(tmpdir, output_file_name)
         post = post_factory(id=1)
@@ -371,11 +372,9 @@ def test_update_post_content_for_new_post(
         assert post.checksum == 'crc'
         assert os.path.exists(output_file_path)
         if post.type in (model.Post.TYPE_IMAGE, model.Post.TYPE_ANIMATION):
-            image_hash.delete_image.assert_called_once_with(post.post_id)
+            assert db.session.query(model.PostSignature).count() == 1
-            image_hash.add_image.assert_called_once_with(post.post_id, content)
         else:
-            image_hash.delete_image.assert_not_called()
+            assert db.session.query(model.PostSignature).count() == 0
-            image_hash.add_image.assert_not_called()
 
 
 def test_update_post_content_to_existing_content(
@@ -388,6 +387,7 @@ def test_update_post_content_to_existing_content(
             'post_height': 300,
         },
         'secret': 'test',
+        'allow_broken_uploads': False,
     })
     post = post_factory()
     another_post = post_factory()
@@ -398,8 +398,10 @@ def test_update_post_content_to_existing_content(
         posts.update_post_content(another_post, read_asset('png.png'))
 
 
+@pytest.mark.parametrize('allow_broken_uploads', [True, False])
 def test_update_post_content_with_broken_content(
-        tmpdir, config_injector, post_factory, read_asset):
+        tmpdir, config_injector, post_factory, read_asset,
+        allow_broken_uploads):
     # the rationale behind this behavior is to salvage user upload even if the
     # server software thinks it's broken. chances are the server is wrong,
     # especially about flash movies.
@@ -410,18 +412,28 @@ def test_update_post_content_with_broken_content(
             'post_height': 300,
         },
         'secret': 'test',
+        'allow_broken_uploads': allow_broken_uploads,
     })
     post = post_factory()
     another_post = post_factory()
     db.session.add_all([post, another_post])
-    posts.update_post_content(post, read_asset('png-broken.png'))
+    if allow_broken_uploads:
-    db.session.flush()
+        posts.update_post_content(post, read_asset('png-broken.png'))
-    assert post.canvas_width is None
+        db.session.flush()
-    assert post.canvas_height is None
+        assert post.canvas_width is None
+        assert post.canvas_height is None
+    else:
+        with pytest.raises(posts.InvalidPostContentError):
+            posts.update_post_content(post, read_asset('png-broken.png'))
+        db.session.flush()
 
 
 @pytest.mark.parametrize('input_content', [None, b'not a media file'])
-def test_update_post_content_with_invalid_content(input_content):
+def test_update_post_content_with_invalid_content(
+        config_injector, input_content):
+    config_injector({
+        'allow_broken_uploads': True,
+    })
     post = model.Post()
     with pytest.raises(posts.InvalidPostContentError):
         posts.update_post_content(post, input_content)
@@ -437,6 +449,7 @@ def test_update_post_thumbnail_to_new_one(
             'post_height': 300,
         },
         'secret': 'test',
+        'allow_broken_uploads': False,
     })
     post = post_factory(id=1)
     db.session.add(post)
@@ -472,6 +485,7 @@ def test_update_post_thumbnail_to_default(
             'post_height': 300,
         },
         'secret': 'test',
+        'allow_broken_uploads': False,
     })
     post = post_factory(id=1)
     db.session.add(post)
@@ -506,6 +520,7 @@ def test_update_post_thumbnail_with_broken_thumbnail(
             'post_height': 300,
         },
         'secret': 'test',
+        'allow_broken_uploads': False,
     })
     post = post_factory(id=1)
     db.session.add(post)
@@ -544,6 +559,7 @@ def test_update_post_content_leaving_custom_thumbnail(
             'post_height': 300,
         },
         'secret': 'test',
+        'allow_broken_uploads': False,
     })
     post = post_factory(id=1)
     db.session.add(post)
@@ -964,3 +980,19 @@ def test_merge_posts_replaces_content(
     assert os.path.exists(source_path)
     assert os.path.exists(target_path1)
     assert not os.path.exists(target_path2)
+
+
+def test_search_by_image(post_factory, config_injector, read_asset):
+    config_injector({'allow_broken_uploads': False})
+    post = post_factory()
+    posts.generate_post_signature(post, read_asset('jpeg.jpg'))
+    db.session.flush()
+
+    result1 = posts.search_by_image(read_asset('jpeg-similar.jpg'))
+    assert len(result1) == 1
+    result1_distance, result1_post = result1[0]
+    assert abs(result1_distance - 0.20599895341812172) < 1e-8
+    assert result1_post.post_id == post.post_id
+
+    result2 = posts.search_by_image(read_asset('png.png'))
+    assert not result2

server/szurubooru/tests/model/test_post.py

@@ -81,7 +81,11 @@ def test_cascade_deletions(
     note.post = post
     note.polygon = ''
     note.text = ''
-    db.session.add_all([score, favorite, feature, note])
+    signature = model.PostSignature()
+    signature.post = post
+    signature.signature = b'testvalue'
+    signature.words = list(range(50))
+    db.session.add_all([score, favorite, feature, note, signature])
     db.session.flush()
 
     post.user = user
@@ -107,6 +111,7 @@ def test_cascade_deletions(
     assert db.session.query(model.PostNote).count() == 1
     assert db.session.query(model.PostFeature).count() == 1
     assert db.session.query(model.PostFavorite).count() == 1
+    assert db.session.query(model.PostSignature).count() == 1
     assert db.session.query(model.Comment).count() == 1
 
     db.session.delete(post)
@@ -122,6 +127,7 @@ def test_cascade_deletions(
     assert db.session.query(model.PostNote).count() == 0
     assert db.session.query(model.PostFeature).count() == 0
     assert db.session.query(model.PostFavorite).count() == 0
+    assert db.session.query(model.PostSignature).count() == 0
     assert db.session.query(model.Comment).count() == 0