Skip to content

Commit

Permalink
chg: dev: api and models
Browse files Browse the repository at this point in the history
- plugin class and func moved to model
  - it can be called from api module
-
  • Loading branch information
rachmadaniHaryono committed Jul 5, 2018
1 parent bc15c3f commit 012f333
Show file tree
Hide file tree
Showing 2 changed files with 119 additions and 92 deletions.
92 changes: 1 addition & 91 deletions gbooru_images_download/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@

from bs4 import BeautifulSoup
from PIL import Image
from yapsy.IPlugin import IPlugin
from yapsy.PluginManager import PluginManager
import requests
import structlog
try:
Expand All @@ -22,6 +20,7 @@

import gbooru_images_download as gid
from . import models, exceptions, plugin
from .models import get_plugin_manager, ModePlugin # NOQA

log = structlog.getLogger(__name__)

Expand Down Expand Up @@ -53,17 +52,6 @@ def sha256_checksum(filename, block_size=65536):
return sha256.hexdigest()


def get_plugin_manager():
manager = PluginManager(plugin_info_ext='ini')
manager.setCategoriesFilter({
'tag_preprocessor': TagPreProcessor,
'mode': ModePlugin,
})
manager.setPluginPlaces([plugin.__path__[0]])
manager.collectPlugins()
return manager


def get_json_response(query, page=1):
url_page = page - 1
url_query = {
Expand Down Expand Up @@ -583,81 +571,3 @@ def create_thumbnail(file_path, thumbnail_folder):
if not os.path.isfile(thumbnail_path):
shutil.copyfile(temp.name, thumbnail_path)
return thumbnail_path


class ModePlugin(IPlugin):
"""Base class for parser plugin."""

def get_match_results(
self, search_term=None, page=1, text=None, response=None, session=None, url=None):
"""Get match result models.
- search_term and page
- text or response or both
"""
raise NotImplementedError

@classmethod
def get_match_results_dict(self, text=None, response=None, session=None, url=None):
"""main function used for plugin.
Returns:
dict: match results data
Examples:
get match results dict.
>>> print(ParserPlugin.get_match_results_dict(text=text)
{
'url': {
'http:example.com/1.html': {
'thumbnail': [
'http:example.com/1.jpg',
'http:example.com/1.png',
...
],
'tag': [
(None, 'tag_value1'),
('namespace1', 'tag_value2', ...)
],
},
...
},
'tag': [(None, 'tag_value3'), ('namespace2', 'tag_value4'), ...]
}
"""
return {}

@classmethod
def match_results_models_from_dict(cls, dict_input, session):
if dict_input['tag']:
pass
for url, data in dict_input['url'].items():
tag_models = []
for nm, tag_value in data['tag']:
# tag_model = models.get_or_create_tag()
if nm:
nm_model = models.get_or_create(session, models.Namespace, value=nm)[0]
tag_models.append(models.get_or_create(
session, models.Tag, value=tag_value, namespace=nm_model)[0])
else:
tag_models.append(models.get_or_create(
session, models.Tag, value=tag_value, namespace=None)[0])
pass
mr_model = None
if data['thumbnail']:
for tu in data['thumbnail']:
mr_model = models.get_or_create_match_result(
session, url=url, thumbnail_url=tu)[0]
yield mr_model
else:
mr_model = models.get_or_create_match_result(session, url=url)[0]
yield mr_model
mr_model.url.tags.extend(tag_models)


class TagPreProcessor(IPlugin):
"""Base class for parser plugin."""

def run_tag_preprocessor(self, tags):
pass
119 changes: 118 additions & 1 deletion gbooru_images_download/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,21 @@
import json
import os

from requests_html import HTMLSession
from flask import flash
from flask_admin.babel import gettext
from flask_sqlalchemy import SQLAlchemy
from requests_html import HTMLSession
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import attributes as orm_attributes, relationship
from sqlalchemy.types import TIMESTAMP
from sqlalchemy_utils.types import ChoiceType, JSONType, ScalarListType, URLType
from yapsy.IPlugin import IPlugin
from yapsy.PluginManager import PluginManager
import requests
import structlog

from . import plugin


log = structlog.getLogger(__name__)
db = SQLAlchemy()
Expand Down Expand Up @@ -91,6 +95,36 @@ def __repr__(self):
'<SearchQuery:{0.id} q:[{0.search_term}] p:{0.page} mode:{1}>'
return templ.format(self, self.mode.name if self.mode else '')

@classmethod
def create(
cls, form, session,
on_model_change_func=None, handle_view_exception=None, after_model_change_func=None
):
try:
model = get_or_create(
session, SearchQuery,
search_term=form.search_term.data, page=form.page.data,
mode=form.mode.data
)[0]
pm = get_plugin_manager()
plugin = pm.getPluginByName(model.mode.name, model.mode.category)
mrs = list(set(plugin.plugin_object.get_match_results(
search_term=model.search_term, page=model.page, session=session)))
model.match_results.extend(mrs)
session.add(model)
if on_model_change_func:
on_model_change_func(form, model, True)
session.commit()
except Exception as ex:
if handle_view_exception and handle_view_exception(ex):
flash(gettext('Failed to create record. %(error)s', error=str(ex)), 'error')
log.exception('Failed to create record.')
session.rollback()
return False
else:
after_model_change_func(form, model, True)
return model


class MatchResult(Base):
"""Match result."""
Expand Down Expand Up @@ -371,3 +405,86 @@ def get_or_create(session, model, **kwargs):
session.add(instance)
created = True
return instance, created

# ## plugin


def get_plugin_manager():
manager = PluginManager(plugin_info_ext='ini')
manager.setCategoriesFilter({
'mode': ModePlugin,
})
manager.setPluginPlaces([plugin.__path__[0]])
manager.collectPlugins()
return manager


class ModePlugin(IPlugin):
"""Base class for parser plugin."""

def get_match_results(
self, search_term=None, page=1, text=None, response=None, session=None, url=None):
"""Get match result models.
- search_term and page
- text or response or both
"""
raise NotImplementedError

@classmethod
def get_match_results_dict(self, text=None, response=None, session=None, url=None):
"""main function used for plugin.
Returns:
dict: match results data
Examples:
get match results dict.
>>> print(ParserPlugin.get_match_results_dict(text=text)
{
'url': {
'http:example.com/1.html': {
'thumbnail': [
'http:example.com/1.jpg',
'http:example.com/1.png',
...
],
'tag': [
(None, 'tag_value1'),
('namespace1', 'tag_value2', ...)
],
},
...
},
'tag': [(None, 'tag_value3'), ('namespace2', 'tag_value4'), ...]
}
"""
return {}

@classmethod
def match_results_models_from_dict(cls, dict_input, session):
if dict_input['tag']:
pass
for url, data in dict_input['url'].items():
tag_models = []
for nm, tag_value in data['tag']:
# tag_model = models.get_or_create_tag()
if nm:
nm_model = get_or_create(session, Namespace, value=nm)[0]
tag_models.append(get_or_create(
session, Tag, value=tag_value, namespace=nm_model)[0])
else:
tag_models.append(get_or_create(
session, Tag, value=tag_value, namespace=None)[0])
pass
mr_model = None
if data['thumbnail']:
for tu in data['thumbnail']:
mr_model = get_or_create_match_result(
session, url=url, thumbnail_url=tu)[0]
yield mr_model
else:
mr_model = get_or_create_match_result(session, url=url)[0]
yield mr_model
mr_model.url.tags.extend(tag_models)

0 comments on commit 012f333

Please sign in to comment.