server/tags: export also tag categories

API.md

@@ -14,7 +14,7 @@
 2. [API reference](#api-reference)
 
     - Tag categories
-        - [Listing tag categories](#listing-tags-categories)
+        - [Listing tag categories](#listing-tag-categories)
         - [Creating tag category](#creating-tag-category)
         - [Updating tag category](#updating-tag-category)
         - [Getting tag category](#getting-tag-category)
@@ -117,6 +117,12 @@ data.
 
     Lists all tag categories. Doesn't support paging.
 
+    **Note**: independently, the server exports current tag category list
+    snapshots to the data directory under `tags.json` name. Its purpose is to
+    reduce the trips frontend needs to make when doing autocompletion, and ease
+    caching. The data directory and its URL are controlled with `data_dir` and
+    `data_url` variables in server's configuration.
+
 
 ## Creating tag category
 - **Request**
@@ -231,6 +237,7 @@ data.
 
     - the tag category does not exist
     - the tag category is used by some tags
+    - the tag category is the last tag category available
     - privileges are too low
 
 - **Description**
@@ -352,7 +359,7 @@ data.
 - **Errors**
 
     - any name is used by an existing tag (names are case insensitive)
-    - any name, implication or suggestion has invalid name
+    - any name, implication or is invalid
     - category is invalid
     - no name was specified
     - implications or suggestions contain any item from names (e.g. there's a
@@ -411,12 +418,12 @@ data.
 
     Updates an existing tag using specified parameters. Names, suggestions and
     implications must match `tag_name_regex` from server's configuration.
-    Category must be one of `tag_categories` from server's configuration.
+    Category must exist and is the same as `name` field within
-    If specified implied tags or suggested tags do not exist yet, they will
+    [`<tag-category>` resource](#tag-category). If specified implied tags or
-    be automatically created. Tags created automatically have no implications,
+    suggested tags do not exist yet, they will be automatically created. Tags
-    no suggestions, one name and their category is set to the first item of
+    created automatically have no implications, no suggestions, one name and
-    `tag_categories` from server's configuration. All fields are optional -
+    their category is set to the first tag category found. All fields are
-    update concerns only provided fields.
+    optional - update concerns only provided fields.
 
 
 ## Getting tag

config.yaml.dist

@@ -99,7 +99,8 @@ privileges:
     'tags:edit:category':      power_user
     'tags:edit:implications':  power_user
     'tags:edit:suggestions':   power_user
-    'tags:list':               regular_user
+    'tags:list':               regular_user # note: will be available as data_url/tags.json anyway
+    'tags:view':               anonymous
     'tags:masstag':            power_user
     'tags:merge':              mod
     'tags:delete':             mod
@@ -107,7 +108,8 @@ privileges:
     'tag_categories:create':        mod
     'tag_categories:edit:name':     mod
     'tag_categories:edit:color':    mod
-    'tag_categories:list':          anonymous
+    'tag_categories:list':          anonymous # note: will be available as data_url/tags.json anyway
+    'tag_categories:view':          anonymous
     'tag_categories:delete':        mod
 
     'comments:create':         regular_user

server/szurubooru/db/tag.py

@@ -47,18 +47,20 @@ class Tag(Base):
     creation_time = Column('creation_time', DateTime, nullable=False)
     last_edit_time = Column('last_edit_time', DateTime)
 
-    category = relationship('TagCategory')
+    category = relationship('TagCategory', lazy='joined')
-    names = relationship('TagName', cascade='all, delete-orphan')
+    names = relationship('TagName', cascade='all, delete-orphan', lazy='joined')
     suggestions = relationship(
         'Tag',
         secondary='tag_suggestion',
         primaryjoin=tag_id == TagSuggestion.parent_id,
-        secondaryjoin=tag_id == TagSuggestion.child_id)
+        secondaryjoin=tag_id == TagSuggestion.child_id,
+        lazy='joined')
     implications = relationship(
         'Tag',
         secondary='tag_implication',
         primaryjoin=tag_id == TagImplication.parent_id,
-        secondaryjoin=tag_id == TagImplication.child_id)
+        secondaryjoin=tag_id == TagImplication.child_id,
+        lazy='joined')
 
     post_count = column_property(
         select([func.count('Post.post_id')]) \

server/szurubooru/search/search_executor.py

@@ -21,13 +21,17 @@ class SearchExecutor(object):
         entities = filter_query \
             .offset((page - 1) * page_size).limit(page_size).all()
         count_query = filter_query.statement \
-            .with_only_columns([sqlalchemy.func.count()]).order_by(None)
+            .with_only_columns([sqlalchemy.func.count()]) \
-        count = filter_query.session.execute(count_query).scalar()
+            .order_by(None)
+        count = filter_query \
+            .session.execute(count_query) \
+            .scalar()
         return (count, entities)
 
     def _prepare(self, query_text):
         ''' Parse input and return SQLAlchemy query. '''
-        query = self._search_config.create_query()
+        query = self._search_config.create_query() \
+            .options(sqlalchemy.orm.lazyload('*'))
         for token in re.split(r'\s+', (query_text or '').lower()):
             if not token:
                 continue

server/szurubooru/tests/api/test_tag_export.py

@@ -4,17 +4,27 @@ import json
 from szurubooru import config, db
 from szurubooru.util import tags
 
-def test_export(tmpdir, session, config_injector, tag_factory):
+def test_export(
+        tmpdir,
+        query_counter,
+        session,
+        config_injector,
+        tag_factory,
+        tag_category_factory):
     config_injector({
         'data_dir': str(tmpdir)
     })
-    sug1 = tag_factory(names=['sug1'])
+    cat1 = tag_category_factory(name='cat1', color='black')
-    sug2 = tag_factory(names=['sug2'])
+    cat2 = tag_category_factory(name='cat2', color='white')
-    imp1 = tag_factory(names=['imp1'])
+    session.add_all([cat1, cat2])
-    imp2 = tag_factory(names=['imp2'])
+    session.flush()
-    tag = tag_factory(names=['alias1', 'alias2'])
+    sug1 = tag_factory(names=['sug1'], category=cat1)
+    sug2 = tag_factory(names=['sug2'], category=cat1)
+    imp1 = tag_factory(names=['imp1'], category=cat1)
+    imp2 = tag_factory(names=['imp2'], category=cat1)
+    tag = tag_factory(names=['alias1', 'alias2'], category=cat2)
     tag.post_count = 1
-    session.add_all([tag, sug1, sug2, imp1, imp2])
+    session.add_all([tag, sug1, sug2, imp1, imp2, cat1, cat2])
     session.flush()
     session.add_all([
         db.TagSuggestion(tag.tag_id, sug1.tag_id),
@@ -24,19 +34,29 @@ def test_export(tmpdir, session, config_injector, tag_factory):
     ])
     session.flush()
 
+    with query_counter:
         tags.export_to_json()
+        assert len(query_counter.statements) == 2
+
     export_path = os.path.join(config.config['data_dir'], 'tags.json')
     assert os.path.exists(export_path)
     with open(export_path, 'r') as handle:
-        assert json.loads(handle.read()) == [
+        assert json.loads(handle.read()) == {
+            'tags': [
                 {
                     'names': ['alias1', 'alias2'],
                     'usages': 1,
+                    'category': 'cat2',
                     'suggestions': ['sug1', 'sug2'],
                     'implications': ['imp1', 'imp2'],
                 },
-            {'names': ['sug1'], 'usages': 0},
+                {'names': ['sug1'], 'usages': 0, 'category': 'cat1'},
-            {'names': ['sug2'], 'usages': 0},
+                {'names': ['sug2'], 'usages': 0, 'category': 'cat1'},
-            {'names': ['imp1'], 'usages': 0},
+                {'names': ['imp1'], 'usages': 0, 'category': 'cat1'},
-            {'names': ['imp2'], 'usages': 0},
+                {'names': ['imp2'], 'usages': 0, 'category': 'cat1'},
+            ],
+            'categories': [
+                {'name': 'cat1', 'color': 'black'},
+                {'name': 'cat2', 'color': 'white'},
             ]
+        }

server/szurubooru/tests/conftest.py

@@ -7,6 +7,32 @@ import sqlalchemy
 from szurubooru import api, config, db
 from szurubooru.util import misc
 
+class QueryCounter(object):
+    def __init__(self):
+        self._statements = []
+
+    def __enter__(self):
+        self._statements = []
+
+    def __exit__(self, *args, **kwargs):
+        self._statements = []
+
+    def create_before_cursor_execute(self):
+        def before_cursor_execute(
+                _conn, _cursor, statement, _parameters, _context, _executemany):
+            self._statements.append(statement)
+        return before_cursor_execute
+
+    @property
+    def statements(self):
+        return self._statements
+
+_query_counter = QueryCounter()
+
+@pytest.fixture
+def query_counter():
+    return _query_counter
+
 def get_unique_name():
     return str(uuid.uuid4())
 
@@ -21,11 +47,15 @@ def fake_datetime():
     return injector
 
 @pytest.yield_fixture
-def session(autoload=True):
+def session(query_counter, autoload=True):
     import logging
     logging.basicConfig()
     logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
     engine = sqlalchemy.create_engine('sqlite:///:memory:')
+    sqlalchemy.event.listen(
+        engine,
+        'before_cursor_execute',
+        query_counter.create_before_cursor_execute())
     session_maker = sqlalchemy.orm.sessionmaker(bind=engine)
     session = sqlalchemy.orm.scoped_session(session_maker)
     db.Base.query = session.query_property()

server/szurubooru/util/tags.py

@@ -28,11 +28,21 @@ def _check_name_intersection(names1, names2):
     return len(set(_lower_list(names1)).intersection(_lower_list(names2))) > 0
 
 def export_to_json():
-    output = []
+    output = {
-    for tag in db.session().query(db.Tag).all():
+        'tags': [],
+        'categories': [],
+    }
+    all_tags = db.session() \
+        .query(db.Tag) \
+        .options(
+            sqlalchemy.orm.joinedload('suggestions'),
+            sqlalchemy.orm.joinedload('implications')) \
+        .all()
+    for tag in all_tags:
         item = {
             'names': [tag_name.name for tag_name in tag.names],
-            'usages': tag.post_count
+            'usages': tag.post_count,
+            'category': tag.category.name,
         }
         if len(tag.suggestions):
             item['suggestions'] = \
@@ -40,7 +50,12 @@ def export_to_json():
         if len(tag.implications):
             item['implications'] = \
                 [rel.names[0].name for rel in tag.implications]
-        output.append(item)
+        output['tags'].append(item)
+    for category in tag_categories.get_all_categories():
+        output['categories'].append({
+            'name': category.name,
+            'color': category.color,
+        })
     export_path = os.path.join(config.config['data_dir'], 'tags.json')
     with open(export_path, 'w') as handle:
         handle.write(json.dumps(output, separators=(',', ':')))