diff --git a/gbooru_images_download/api.py b/gbooru_images_download/api.py index 4e711efe..20781b7b 100644 --- a/gbooru_images_download/api.py +++ b/gbooru_images_download/api.py @@ -9,8 +9,6 @@ from bs4 import BeautifulSoup from PIL import Image -from yapsy.IPlugin import IPlugin -from yapsy.PluginManager import PluginManager import requests import structlog try: @@ -22,6 +20,7 @@ import gbooru_images_download as gid from . import models, exceptions, plugin +from .models import get_plugin_manager, ModePlugin # NOQA log = structlog.getLogger(__name__) @@ -53,17 +52,6 @@ def sha256_checksum(filename, block_size=65536): return sha256.hexdigest() -def get_plugin_manager(): - manager = PluginManager(plugin_info_ext='ini') - manager.setCategoriesFilter({ - 'tag_preprocessor': TagPreProcessor, - 'mode': ModePlugin, - }) - manager.setPluginPlaces([plugin.__path__[0]]) - manager.collectPlugins() - return manager - - def get_json_response(query, page=1): url_page = page - 1 url_query = { @@ -583,81 +571,3 @@ def create_thumbnail(file_path, thumbnail_folder): if not os.path.isfile(thumbnail_path): shutil.copyfile(temp.name, thumbnail_path) return thumbnail_path - - -class ModePlugin(IPlugin): - """Base class for parser plugin.""" - - def get_match_results( - self, search_term=None, page=1, text=None, response=None, session=None, url=None): - """Get match result models. - - - search_term and page - - text or response or both - """ - raise NotImplementedError - - @classmethod - def get_match_results_dict(self, text=None, response=None, session=None, url=None): - """main function used for plugin. - - Returns: - dict: match results data - - Examples: - get match results dict. - - >>> print(ParserPlugin.get_match_results_dict(text=text) - { - 'url': { - 'http:example.com/1.html': { - 'thumbnail': [ - 'http:example.com/1.jpg', - 'http:example.com/1.png', - ... - ], - 'tag': [ - (None, 'tag_value1'), - ('namespace1', 'tag_value2', ...) - ], - }, - ... - }, - 'tag': [(None, 'tag_value3'), ('namespace2', 'tag_value4'), ...] - } - """ - return {} - - @classmethod - def match_results_models_from_dict(cls, dict_input, session): - if dict_input['tag']: - pass - for url, data in dict_input['url'].items(): - tag_models = [] - for nm, tag_value in data['tag']: - # tag_model = models.get_or_create_tag() - if nm: - nm_model = models.get_or_create(session, models.Namespace, value=nm)[0] - tag_models.append(models.get_or_create( - session, models.Tag, value=tag_value, namespace=nm_model)[0]) - else: - tag_models.append(models.get_or_create( - session, models.Tag, value=tag_value, namespace=None)[0]) - pass - mr_model = None - if data['thumbnail']: - for tu in data['thumbnail']: - mr_model = models.get_or_create_match_result( - session, url=url, thumbnail_url=tu)[0] - yield mr_model - else: - mr_model = models.get_or_create_match_result(session, url=url)[0] - yield mr_model - mr_model.url.tags.extend(tag_models) - - -class TagPreProcessor(IPlugin): - """Base class for parser plugin.""" - - def run_tag_preprocessor(self, tags): - pass diff --git a/gbooru_images_download/models.py b/gbooru_images_download/models.py index 061228ce..fe7005b7 100644 --- a/gbooru_images_download/models.py +++ b/gbooru_images_download/models.py @@ -5,17 +5,21 @@ import json import os -from requests_html import HTMLSession from flask import flash from flask_admin.babel import gettext from flask_sqlalchemy import SQLAlchemy +from requests_html import HTMLSession from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import attributes as orm_attributes, relationship from sqlalchemy.types import TIMESTAMP from sqlalchemy_utils.types import ChoiceType, JSONType, ScalarListType, URLType +from yapsy.IPlugin import IPlugin +from yapsy.PluginManager import PluginManager import requests import structlog +from . import plugin + log = structlog.getLogger(__name__) db = SQLAlchemy() @@ -91,6 +95,36 @@ def __repr__(self): '' return templ.format(self, self.mode.name if self.mode else '') + @classmethod + def create( + cls, form, session, + on_model_change_func=None, handle_view_exception=None, after_model_change_func=None + ): + try: + model = get_or_create( + session, SearchQuery, + search_term=form.search_term.data, page=form.page.data, + mode=form.mode.data + )[0] + pm = get_plugin_manager() + plugin = pm.getPluginByName(model.mode.name, model.mode.category) + mrs = list(set(plugin.plugin_object.get_match_results( + search_term=model.search_term, page=model.page, session=session))) + model.match_results.extend(mrs) + session.add(model) + if on_model_change_func: + on_model_change_func(form, model, True) + session.commit() + except Exception as ex: + if handle_view_exception and handle_view_exception(ex): + flash(gettext('Failed to create record. %(error)s', error=str(ex)), 'error') + log.exception('Failed to create record.') + session.rollback() + return False + else: + after_model_change_func(form, model, True) + return model + class MatchResult(Base): """Match result.""" @@ -371,3 +405,86 @@ def get_or_create(session, model, **kwargs): session.add(instance) created = True return instance, created + +# ## plugin + + +def get_plugin_manager(): + manager = PluginManager(plugin_info_ext='ini') + manager.setCategoriesFilter({ + 'mode': ModePlugin, + }) + manager.setPluginPlaces([plugin.__path__[0]]) + manager.collectPlugins() + return manager + + +class ModePlugin(IPlugin): + """Base class for parser plugin.""" + + def get_match_results( + self, search_term=None, page=1, text=None, response=None, session=None, url=None): + """Get match result models. + + - search_term and page + - text or response or both + """ + raise NotImplementedError + + @classmethod + def get_match_results_dict(self, text=None, response=None, session=None, url=None): + """main function used for plugin. + + Returns: + dict: match results data + + Examples: + get match results dict. + + >>> print(ParserPlugin.get_match_results_dict(text=text) + { + 'url': { + 'http:example.com/1.html': { + 'thumbnail': [ + 'http:example.com/1.jpg', + 'http:example.com/1.png', + ... + ], + 'tag': [ + (None, 'tag_value1'), + ('namespace1', 'tag_value2', ...) + ], + }, + ... + }, + 'tag': [(None, 'tag_value3'), ('namespace2', 'tag_value4'), ...] + } + """ + return {} + + @classmethod + def match_results_models_from_dict(cls, dict_input, session): + if dict_input['tag']: + pass + for url, data in dict_input['url'].items(): + tag_models = [] + for nm, tag_value in data['tag']: + # tag_model = models.get_or_create_tag() + if nm: + nm_model = get_or_create(session, Namespace, value=nm)[0] + tag_models.append(get_or_create( + session, Tag, value=tag_value, namespace=nm_model)[0]) + else: + tag_models.append(get_or_create( + session, Tag, value=tag_value, namespace=None)[0]) + pass + mr_model = None + if data['thumbnail']: + for tu in data['thumbnail']: + mr_model = get_or_create_match_result( + session, url=url, thumbnail_url=tu)[0] + yield mr_model + else: + mr_model = get_or_create_match_result(session, url=url)[0] + yield mr_model + mr_model.url.tags.extend(tag_models)