ulysia/szurubooru
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server/tags: improve tag export performance

Browse source 
2000 ms --> 250 ms (including overhead of unknown size).
Not bad if you ask me.
This commit is contained in:
rr- 2016-07-26 23:04:53 +02:00
parent 3f2717843a
commit 8ef0017531
2 changed files with 49 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

71
server/szurubooru/func/tags.py
View file

@ -55,34 +55,51 @@ def serialize_tag(tag, options=None):
options) options)
def export_to_json(): def export_to_json():
output = { tags = {}
'tags': [], categories = {}
'categories': [],
} for result in db.session.query(
all_tags = db.session \ db.TagCategory.tag_category_id,
.query(db.Tag) \ db.TagCategory.name,
.options( db.TagCategory.color).all():
sqlalchemy.orm.joinedload('suggestions'), categories[result[0]] = {
sqlalchemy.orm.joinedload('implications')) \ 'name': result[1],
.all() 'color': result[2],
for tag in all_tags:
item = {
'names': [tag_name.name for tag_name in tag.names],
'usages': tag.post_count,
'category': tag.category.name,
} }
if len(tag.suggestions):
item['suggestions'] = \ for result in db.session.query(db.TagName.tag_id, db.TagName.name).all():
[rel.names[0].name for rel in tag.suggestions] if not result[0] in tags:
if len(tag.implications): tags[result[0]] = {'names': []}
item['implications'] = \ tags[result[0]]['names'].append(result[1])
[rel.names[0].name for rel in tag.implications]
output['tags'].append(item) for result in db.session \
for category in tag_categories.get_all_categories(): .query(db.TagSuggestion.parent_id, db.TagName.name) \
output['categories'].append({ .join(db.TagName, db.TagName.tag_id == db.TagSuggestion.child_id) \
'name': category.name, .all():
'color': category.color, if not 'suggestions' in tags[result[0]]:
}) tags[result[0]]['suggestions'] = []
tags[result[0]]['suggestions'].append(result[1])
for result in db.session \
.query(db.TagImplication.parent_id, db.TagName.name) \
.join(db.TagName, db.TagName.tag_id == db.TagImplication.child_id) \
.all():
if not 'implications' in tags[result[0]]:
tags[result[0]]['implications'] = []
tags[result[0]]['implications'].append(result[1])
for result in db.session.query(
db.Tag.tag_id,
db.Tag.category_id,
db.Tag.post_count).all():
tags[result[0]]['category'] = categories[result[1]]['name']
tags[result[0]]['usages'] = result[2]
output = {
'categories': list(categories.values()),
'tags': list(tags.values()),
}
export_path = os.path.join(config.config['data_dir'], 'tags.json') export_path = os.path.join(config.config['data_dir'], 'tags.json')
with open(export_path, 'w') as handle: with open(export_path, 'w') as handle:
handle.write(json.dumps(output, separators=(',', ':'))) handle.write(json.dumps(output, separators=(',', ':')))

7
server/szurubooru/tests/api/test_tag_export.py
View file

@ -8,6 +8,7 @@ def test_export(
tmpdir, tmpdir,
query_counter, query_counter,
config_injector, config_injector,
post_factory,
tag_factory, tag_factory,
tag_category_factory): tag_category_factory):
config_injector({ config_injector({
@ -22,10 +23,12 @@ def test_export(
imp1 = tag_factory(names=['imp1'], category=cat1) imp1 = tag_factory(names=['imp1'], category=cat1)
imp2 = tag_factory(names=['imp2'], category=cat1) imp2 = tag_factory(names=['imp2'], category=cat1)
tag = tag_factory(names=['alias1', 'alias2'], category=cat2) tag = tag_factory(names=['alias1', 'alias2'], category=cat2)
tag.post_count = 1
db.session.add_all([tag, sug1, sug2, imp1, imp2, cat1, cat2]) db.session.add_all([tag, sug1, sug2, imp1, imp2, cat1, cat2])
post = post_factory()
post.tags = [tag]
db.session.flush() db.session.flush()
db.session.add_all([ db.session.add_all([
post,
db.TagSuggestion(tag.tag_id, sug1.tag_id), db.TagSuggestion(tag.tag_id, sug1.tag_id),
db.TagSuggestion(tag.tag_id, sug2.tag_id), db.TagSuggestion(tag.tag_id, sug2.tag_id),
db.TagImplication(tag.tag_id, imp1.tag_id), db.TagImplication(tag.tag_id, imp1.tag_id),
@ -35,7 +38,7 @@ def test_export(
with query_counter: with query_counter:
tags.export_to_json() tags.export_to_json()
assert len(query_counter.statements) == 2 assert len(query_counter.statements) == 5
export_path = os.path.join(config.config['data_dir'], 'tags.json') export_path = os.path.join(config.config['data_dir'], 'tags.json')
assert os.path.exists(export_path) assert os.path.exists(export_path)