From bd9284b7f8d809f122f68f04525244457c90c645 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Sat, 7 Mar 2020 21:02:01 -0500 Subject: [PATCH] server/tests: update unit tests for elasticsearch removal --- .../tests/api/test_post_creating.py | 4 +- .../tests/api/test_post_updating.py | 1 + server/szurubooru/tests/conftest.py | 9 +--- .../szurubooru/tests/func/test_image_hash.py | 47 +++++++---------- server/szurubooru/tests/func/test_posts.py | 52 +++++++++++++++---- server/szurubooru/tests/model/test_post.py | 8 ++- 6 files changed, 71 insertions(+), 50 deletions(-) diff --git a/server/szurubooru/tests/api/test_post_creating.py b/server/szurubooru/tests/api/test_post_creating.py index c7088978..9d94a929 100644 --- a/server/szurubooru/tests/api/test_post_creating.py +++ b/server/szurubooru/tests/api/test_post_creating.py @@ -12,6 +12,7 @@ def inject_config(config_injector): 'posts:create:identified': model.User.RANK_REGULAR, 'tags:create': model.User.RANK_REGULAR, }, + 'allow_broken_uploads': False, }) @@ -250,8 +251,7 @@ def test_omitting_optional_field( def test_errors_not_spending_ids( - config_injector, tmpdir, context_factory, read_asset, user_factory, - skip_post_hashing): + config_injector, tmpdir, context_factory, read_asset, user_factory): config_injector({ 'data_dir': str(tmpdir.mkdir('data')), 'data_url': 'example.com', diff --git a/server/szurubooru/tests/api/test_post_updating.py b/server/szurubooru/tests/api/test_post_updating.py index fa298eaa..229ecef7 100644 --- a/server/szurubooru/tests/api/test_post_updating.py +++ b/server/szurubooru/tests/api/test_post_updating.py @@ -19,6 +19,7 @@ def inject_config(config_injector): 'posts:edit:thumbnail': model.User.RANK_REGULAR, 'tags:create': model.User.RANK_MODERATOR, }, + 'allow_broken_uploads': False, }) diff --git a/server/szurubooru/tests/conftest.py b/server/szurubooru/tests/conftest.py index 593771d8..a131fece 100644 --- a/server/szurubooru/tests/conftest.py +++ b/server/szurubooru/tests/conftest.py @@ -139,15 +139,8 @@ def tag_factory(): return factory -@pytest.yield_fixture -def skip_post_hashing(): - with patch('szurubooru.func.image_hash.add_image'), \ - patch('szurubooru.func.image_hash.delete_image'): - yield - - @pytest.fixture -def post_factory(skip_post_hashing): +def post_factory(): # pylint: disable=invalid-name def factory( id=None, diff --git a/server/szurubooru/tests/func/test_image_hash.py b/server/szurubooru/tests/func/test_image_hash.py index ce82477c..10252915 100644 --- a/server/szurubooru/tests/func/test_image_hash.py +++ b/server/szurubooru/tests/func/test_image_hash.py @@ -1,37 +1,26 @@ import pytest from szurubooru.func import image_hash +from numpy import array_equal -def test_hashing(read_asset, config_injector): - config_injector({ - 'elasticsearch': { - 'host': 'localhost', - 'port': 9200, - 'index': 'szurubooru_test', - 'user': 'szurubooru', - 'pass': None, - }, - }) +def test_signature_functions(read_asset, config_injector): + sig1 = image_hash.generate_signature(read_asset('jpeg.jpg')) + sig2 = image_hash.generate_signature(read_asset('jpeg-similar.jpg')) - if not image_hash.get_session().ping(): - pytest.xfail( - 'Unable to connect to ElasticSearch, ' - 'perhaps it is not available for this test?') + sig1_repacked = image_hash.unpack_signature( + image_hash.pack_signature(sig1)) + sig2_repacked = image_hash.unpack_signature( + image_hash.pack_signature(sig2)) + assert array_equal(sig1, sig1_repacked) + assert array_equal(sig2, sig2_repacked) - image_hash.purge() - image_hash.add_image('test', read_asset('jpeg.jpg')) + dist1 = image_hash.normalized_distance([sig1], sig2) + assert abs(dist1[0] - 0.20599895341812172) < 1e-8 - paths = image_hash.get_all_paths() - results_exact = image_hash.search_by_image(read_asset('jpeg.jpg')) - results_similar = image_hash.search_by_image( - read_asset('jpeg-similar.jpg')) + dist2 = image_hash.normalized_distance([sig2], sig2) + assert abs(dist2[0]) < 1e-8 - assert len(paths) == 1 - assert len(results_exact) == 1 - assert len(results_similar) == 1 - assert results_exact[0].path == 'test' - assert results_exact[0].score == 63 - assert results_exact[0].distance == 0 - assert results_similar[0].path == 'test' - assert results_similar[0].score == 17 - assert abs(results_similar[0].distance - 0.20599895341812172) < 1e-8 + words1 = image_hash.generate_words(sig1) + words2 = image_hash.generate_words(sig2) + words_match = sum(word1 == word2 for word1, word2 in zip(words1, words2)) + assert words_match == 17 diff --git a/server/szurubooru/tests/func/test_posts.py b/server/szurubooru/tests/func/test_posts.py index fc4cfc9f..097136c2 100644 --- a/server/szurubooru/tests/func/test_posts.py +++ b/server/szurubooru/tests/func/test_posts.py @@ -354,6 +354,7 @@ def test_update_post_content_for_new_post( 'post_height': 300, }, 'secret': 'test', + 'allow_broken_uploads': False, }) output_file_path = '{}/data/posts/{}'.format(tmpdir, output_file_name) post = post_factory(id=1) @@ -371,11 +372,9 @@ def test_update_post_content_for_new_post( assert post.checksum == 'crc' assert os.path.exists(output_file_path) if post.type in (model.Post.TYPE_IMAGE, model.Post.TYPE_ANIMATION): - image_hash.delete_image.assert_called_once_with(post.post_id) - image_hash.add_image.assert_called_once_with(post.post_id, content) + assert db.session.query(model.PostSignature).count() == 1 else: - image_hash.delete_image.assert_not_called() - image_hash.add_image.assert_not_called() + assert db.session.query(model.PostSignature).count() == 0 def test_update_post_content_to_existing_content( @@ -388,6 +387,7 @@ def test_update_post_content_to_existing_content( 'post_height': 300, }, 'secret': 'test', + 'allow_broken_uploads': False, }) post = post_factory() another_post = post_factory() @@ -398,8 +398,10 @@ def test_update_post_content_to_existing_content( posts.update_post_content(another_post, read_asset('png.png')) +@pytest.mark.parametrize('allow_broken_uploads', [True, False]) def test_update_post_content_with_broken_content( - tmpdir, config_injector, post_factory, read_asset): + tmpdir, config_injector, post_factory, read_asset, + allow_broken_uploads): # the rationale behind this behavior is to salvage user upload even if the # server software thinks it's broken. chances are the server is wrong, # especially about flash movies. @@ -410,18 +412,28 @@ def test_update_post_content_with_broken_content( 'post_height': 300, }, 'secret': 'test', + 'allow_broken_uploads': allow_broken_uploads, }) post = post_factory() another_post = post_factory() db.session.add_all([post, another_post]) - posts.update_post_content(post, read_asset('png-broken.png')) - db.session.flush() - assert post.canvas_width is None - assert post.canvas_height is None + if allow_broken_uploads: + posts.update_post_content(post, read_asset('png-broken.png')) + db.session.flush() + assert post.canvas_width is None + assert post.canvas_height is None + else: + with pytest.raises(posts.InvalidPostContentError): + posts.update_post_content(post, read_asset('png-broken.png')) + db.session.flush() @pytest.mark.parametrize('input_content', [None, b'not a media file']) -def test_update_post_content_with_invalid_content(input_content): +def test_update_post_content_with_invalid_content( + config_injector, input_content): + config_injector({ + 'allow_broken_uploads': True, + }) post = model.Post() with pytest.raises(posts.InvalidPostContentError): posts.update_post_content(post, input_content) @@ -437,6 +449,7 @@ def test_update_post_thumbnail_to_new_one( 'post_height': 300, }, 'secret': 'test', + 'allow_broken_uploads': False, }) post = post_factory(id=1) db.session.add(post) @@ -472,6 +485,7 @@ def test_update_post_thumbnail_to_default( 'post_height': 300, }, 'secret': 'test', + 'allow_broken_uploads': False, }) post = post_factory(id=1) db.session.add(post) @@ -506,6 +520,7 @@ def test_update_post_thumbnail_with_broken_thumbnail( 'post_height': 300, }, 'secret': 'test', + 'allow_broken_uploads': False, }) post = post_factory(id=1) db.session.add(post) @@ -544,6 +559,7 @@ def test_update_post_content_leaving_custom_thumbnail( 'post_height': 300, }, 'secret': 'test', + 'allow_broken_uploads': False, }) post = post_factory(id=1) db.session.add(post) @@ -964,3 +980,19 @@ def test_merge_posts_replaces_content( assert os.path.exists(source_path) assert os.path.exists(target_path1) assert not os.path.exists(target_path2) + + +def test_search_by_image(post_factory, config_injector, read_asset): + config_injector({'allow_broken_uploads': False}) + post = post_factory() + posts.generate_post_signature(post, read_asset('jpeg.jpg')) + db.session.flush() + + result1 = posts.search_by_image(read_asset('jpeg-similar.jpg')) + assert len(result1) == 1 + result1_distance, result1_post = result1[0] + assert abs(result1_distance - 0.20599895341812172) < 1e-8 + assert result1_post.post_id == post.post_id + + result2 = posts.search_by_image(read_asset('png.png')) + assert not result2 diff --git a/server/szurubooru/tests/model/test_post.py b/server/szurubooru/tests/model/test_post.py index ee691460..75bcbae5 100644 --- a/server/szurubooru/tests/model/test_post.py +++ b/server/szurubooru/tests/model/test_post.py @@ -81,7 +81,11 @@ def test_cascade_deletions( note.post = post note.polygon = '' note.text = '' - db.session.add_all([score, favorite, feature, note]) + signature = model.PostSignature() + signature.post = post + signature.signature = b'testvalue' + signature.words = list(range(50)) + db.session.add_all([score, favorite, feature, note, signature]) db.session.flush() post.user = user @@ -107,6 +111,7 @@ def test_cascade_deletions( assert db.session.query(model.PostNote).count() == 1 assert db.session.query(model.PostFeature).count() == 1 assert db.session.query(model.PostFavorite).count() == 1 + assert db.session.query(model.PostSignature).count() == 1 assert db.session.query(model.Comment).count() == 1 db.session.delete(post) @@ -122,6 +127,7 @@ def test_cascade_deletions( assert db.session.query(model.PostNote).count() == 0 assert db.session.query(model.PostFeature).count() == 0 assert db.session.query(model.PostFavorite).count() == 0 + assert db.session.query(model.PostSignature).count() == 0 assert db.session.query(model.Comment).count() == 0