Refactoring: move all views into the views folder

scrapydweb/__init__.py

@@ -12,7 +12,7 @@
 
 from .__version__ import __url__, __version__
 from .common import handle_metadata
-from .database import Metadata, db
+from .models import Metadata, db
 from .vars import PYTHON_VERSION, SQLALCHEMY_BINDS, SQLALCHEMY_DATABASE_URI
 # from .utils.scheduler import scheduler
 
@@ -158,15 +158,15 @@ def register_view(view, endpoint, url_defaults_list):
         ('api/<opt>', dict(project=None, version_spider_job=None))
     ])
 
-    from .myview import MetadataView
+    from .views.myview import MetadataView
     register_view(MetadataView, 'metadata', [('metadata', None)])
 
     # Overview
-    from .overview.jobs import JobsView, JobsXhrView
+    from .views.overview.jobs import JobsView, JobsXhrView
     register_view(JobsView, 'jobs', [('jobs', None)])
     register_view(JobsXhrView, 'jobs.xhr', [('jobs/xhr/<action>/<int:id>', None)])
 
-    from .overview.servers import ServersView
+    from .views.overview.servers import ServersView
     register_view(ServersView, 'servers', [
         ('servers/<opt>/<project>/<version_job>/<spider>', None),
         ('servers/<opt>/<project>/<version_job>', dict(spider=None)),
@@ -175,13 +175,13 @@ def register_view(view, endpoint, url_defaults_list):
         ('servers', dict(opt=None, project=None, version_job=None, spider=None))
     ])
 
-    from .overview.multinode import MultinodeView
+    from .views.overview.multinode import MultinodeView
     register_view(MultinodeView, 'multinode', [
         ('multinode/<opt>/<project>/<version_job>', None),
         ('multinode/<opt>/<project>', dict(version_job=None))
     ])
 
-    from .overview.tasks import TasksView, TasksXhrView
+    from .views.overview.tasks import TasksView, TasksXhrView
     register_view(TasksView, 'tasks', [
         ('tasks/<int:task_id>/<int:task_result_id>', None),
         ('tasks/<int:task_id>', dict(task_result_id=None)),
@@ -193,16 +193,16 @@ def register_view(view, endpoint, url_defaults_list):
         ('tasks/xhr/<action>', dict(task_id=None, task_result_id=None))
     ])
 
-    from .overview.tasks import bp as bp_tasks_history
+    from .views.overview.tasks import bp as bp_tasks_history
     app.register_blueprint(bp_tasks_history)
 
     # Operations
-    from .operations.deploy import DeployView, DeployUploadView, DeployXhrView
+    from .views.operations.deploy import DeployView, DeployUploadView, DeployXhrView
     register_view(DeployView, 'deploy', [('deploy', None)])
     register_view(DeployUploadView, 'deploy.upload', [('deploy/upload', None)])
     register_view(DeployXhrView, 'deploy.xhr', [('deploy/xhr/<eggname>/<project>/<version>', None)])
 
-    from .operations.schedule import ScheduleView, ScheduleCheckView, ScheduleRunView, ScheduleXhrView, ScheduleTaskView
+    from .views.operations.schedule import ScheduleView, ScheduleCheckView, ScheduleRunView, ScheduleXhrView, ScheduleTaskView
     register_view(ScheduleView, 'schedule', [
         ('schedule/<project>/<version>/<spider>', None),
         ('schedule/<project>/<version>', dict(spider=None)),
@@ -214,43 +214,43 @@ def register_view(view, endpoint, url_defaults_list):
     register_view(ScheduleXhrView, 'schedule.xhr', [('schedule/xhr/<filename>', None)])
     register_view(ScheduleTaskView, 'schedule.task', [('schedule/task', None)])
 
-    from .operations.schedule import bp as bp_schedule_history
+    from .views.operations.schedule import bp as bp_schedule_history
     app.register_blueprint(bp_schedule_history)
 
     # Files
-    from .files.log import LogView
+    from .views.files.log import LogView
     register_view(LogView, 'log', [('log/<opt>/<project>/<spider>/<job>', None)])
 
-    from .files.logs import LogsView
+    from .views.files.logs import LogsView
     register_view(LogsView, 'logs', [
         ('logs/<project>/<spider>', None),
         ('logs/<project>', dict(spider=None)),
         ('logs', dict(project=None, spider=None))
     ])
 
-    from .files.items import ItemsView
+    from .views.files.items import ItemsView
     register_view(ItemsView, 'items', [
         ('items/<project>/<spider>', None),
         ('items/<project>', dict(spider=None)),
         ('items', dict(project=None, spider=None))
     ])
 
-    from .files.projects import ProjectsView
+    from .views.files.projects import ProjectsView
     register_view(ProjectsView, 'projects', [
         ('projects/<opt>/<project>/<version_spider_job>', None),
         ('projects/<opt>/<project>', dict(version_spider_job=None)),
         ('projects', dict(opt='listprojects', project=None, version_spider_job=None))
     ])
 
-    from .files.parse import UploadLogView, UploadedLogView
+    from .views.files.parse import UploadLogView, UploadedLogView
     register_view(UploadLogView, 'parse.upload', [('parse/upload', None)])
     register_view(UploadedLogView, 'parse.uploaded', [('parse/uploaded/<filename>', None)])
 
-    from .files.parse import bp as bp_parse_source
+    from .views.files.parse import bp as bp_parse_source
     app.register_blueprint(bp_parse_source)
 
     # System
-    from .system.settings import SettingsView
+    from .views.system.settings import SettingsView
     register_view(SettingsView, 'settings', [('settings', None)])
 
 

scrapydweb/common.py

@@ -12,7 +12,7 @@
 from requests.auth import _basic_auth_str
 
 from .__version__ import __version__
-from .database import Metadata, db
+from .models import Metadata, db
 
 
 session = requests.Session()

scrapydweb/database.py → scrapydweb/models.py

@@ -70,7 +70,7 @@ def __repr__(self):
 
     return Job
     # sqlalchemy/ext/declarative/clsregistry.py:128: SAWarning: This declarative base already contains a class
-    # with the same class name and module name as scrapydweb.database.Job,
+    # with the same class name and module name as scrapydweb.models.Job,
     # and will be replaced in the string-lookup table.
     # https://stackoverflow.com/questions/27773489/dynamically-create-a-python-subclass-in-a-function
     # return type('Job_%s' % server, (Job, ), dict(__tablename__=server,  __bind_key__='jobs'))

scrapydweb/views/api.py

@@ -2,7 +2,7 @@
 import re
 import time
 
-from ..myview import MyView
+from .myview import MyView
 
 
 API_MAP = dict(start='schedule', stop='cancel', forcestop='cancel', liststats='logs/stats')

scrapydweb/files/items.py → scrapydweb/views/files/items.py

@@ -4,8 +4,8 @@
 
 from flask import render_template, url_for
 
+from ...vars import DIRECTORY_KEYS, DIRECTORY_PATTERN, HREF_NAME_PATTERN
 from ..myview import MyView
-from ..vars import DIRECTORY_KEYS, DIRECTORY_PATTERN, HREF_NAME_PATTERN
 
 
 class ItemsView(MyView):

scrapydweb/files/log.py → scrapydweb/views/files/log.py

@@ -14,10 +14,10 @@
 from flask import flash, render_template, request, url_for
 from logparser import parse
 
+from ...vars import CWD as root_dir
 from ..myview import MyView
 
 
-CWD = os.path.dirname(os.path.abspath(__file__))
 EMAIL_CONTENT_KEYS = [
     'log_critical_count',
     'log_error_count',
@@ -447,7 +447,7 @@ def handle_email_flag(self):
 
                 args = [
                     sys.executable,
-                    os.path.join(os.path.dirname(CWD), 'utils', 'send_email.py'),
+                    os.path.join(root_dir, 'utils', 'send_email.py'),
                     self.json_dumps(self.EMAIL_KWARGS, ensure_ascii=True)
                 ]
                 self.logger.info("Sending email: %s", self.EMAIL_KWARGS['subject'])

scrapydweb/files/logs.py → scrapydweb/views/files/logs.py

@@ -3,8 +3,8 @@
 
 from flask import render_template, url_for
 
+from ...vars import DIRECTORY_KEYS, DIRECTORY_PATTERN, HREF_NAME_PATTERN
 from ..myview import MyView
-from ..vars import DIRECTORY_KEYS, DIRECTORY_PATTERN, HREF_NAME_PATTERN
 
 
 class LogsView(MyView):

scrapydweb/files/parse.py → scrapydweb/views/files/parse.py

@@ -7,8 +7,8 @@
 from logparser import parse
 from werkzeug.utils import secure_filename
 
+from ...vars import PARSE_PATH
 from ..myview import MyView
-from ..vars import PARSE_PATH
 
 
 ALLOWED_EXTENSIONS = {'log', 'txt'}

scrapydweb/views/index.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 from flask import redirect, url_for
 
-from ..myview import MyView
+from .myview import MyView
 
 
 class IndexView(MyView):

scrapydweb/myview.py → scrapydweb/views/myview.py

@@ -8,11 +8,11 @@
 from flask.views import View
 from logparser import __version__ as LOGPARSER_VERSION
 
-from .__version__ import __version__ as SCRAPYDWEB_VERSION
-from .common import get_now_string, get_response_from_view, handle_metadata, handle_slash, json_dumps, session
-from .vars import (ALLOWED_SCRAPYD_LOG_EXTENSIONS, DEMO_PROJECTS_PATH, DEPLOY_PATH, EMAIL_TRIGGER_KEYS, PARSE_PATH,
-                   LEGAL_NAME_PATTERN, SCHEDULE_PATH, STATE_PAUSED, STATE_RUNNING, STATS_PATH, STRICT_NAME_PATTERN)
-from .utils.scheduler import scheduler
+from ..__version__ import __version__ as SCRAPYDWEB_VERSION
+from ..common import get_now_string, get_response_from_view, handle_metadata, handle_slash, json_dumps, session
+from ..vars import (ALLOWED_SCRAPYD_LOG_EXTENSIONS, DEMO_PROJECTS_PATH, DEPLOY_PATH, EMAIL_TRIGGER_KEYS, PARSE_PATH,
+                    LEGAL_NAME_PATTERN, SCHEDULE_PATH, STATE_PAUSED, STATE_RUNNING, STATS_PATH, STRICT_NAME_PATTERN)
+from ..utils.scheduler import scheduler
 
 
 class MyView(View):

scrapydweb/operations/deploy.py → scrapydweb/views/operations/deploy.py

@@ -16,10 +16,10 @@
 from six.moves.configparser import Error as ScrapyCfgParseError
 from werkzeug.utils import secure_filename
 
+from ...vars import PY2
 from ..myview import MyView
 from .scrapyd_deploy import _build_egg, get_config
 from .utils import mkdir_p, slot
-from ..vars import PY2
 
 
 SCRAPY_CFG = """

scrapydweb/operations/execute_task.py → scrapydweb/views/operations/execute_task.py

@@ -5,9 +5,9 @@
 import time
 import traceback
 
-from ..common import get_now_string, get_response_from_view, handle_metadata
-from ..database import Task, TaskResult, TaskJobResult, db
-from ..utils.scheduler import scheduler
+from ...common import get_now_string, get_response_from_view, handle_metadata
+from ...models import Task, TaskResult, TaskJobResult, db
+from ...utils.scheduler import scheduler
 
 
 apscheduler_logger = logging.getLogger('apscheduler')

scrapydweb/operations/schedule.py → scrapydweb/views/operations/schedule.py

@@ -12,11 +12,11 @@
 
 from flask import Blueprint, redirect, render_template, request, send_file, url_for
 
-from ..database import Task, db
+from ...models import Task, db
+from ...vars import RUN_SPIDER_HISTORY_LOG, UA_DICT
 from ..myview import MyView
-from ..vars import RUN_SPIDER_HISTORY_LOG, UA_DICT
-from .utils import slot
 from .execute_task import execute_task
+from .utils import slot
 
 
 apscheduler_logger = logging.getLogger('apscheduler')

scrapydweb/overview/jobs.py → scrapydweb/views/overview/jobs.py

@@ -7,8 +7,8 @@
 from flask import flash, get_flashed_messages, render_template, request, url_for
 from six.moves.urllib.parse import urljoin
 
-from ..common import handle_metadata
-from ..database import create_jobs_table, db
+from ...common import handle_metadata
+from ...models import create_jobs_table, db
 from ..myview import MyView
 
 

scrapydweb/overview/servers.py → scrapydweb/views/overview/servers.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 from flask import flash, render_template, url_for
 
-from ..common import handle_metadata
+from ...common import handle_metadata
 from ..myview import MyView
 
 

scrapydweb/overview/tasks.py → scrapydweb/views/overview/tasks.py

@@ -6,10 +6,10 @@
 
 from flask import Blueprint, flash, render_template, request, send_file, url_for
 
-from ..common import handle_metadata
-from ..database import Task, TaskResult, TaskJobResult, db
+from ...common import handle_metadata
+from ...models import Task, TaskResult, TaskJobResult, db
+from ...vars import SCHEDULER_STATE_DICT, STATE_PAUSED, STATE_RUNNING, TIMER_TASKS_HISTORY_LOG
 from ..myview import MyView
-from ..vars import SCHEDULER_STATE_DICT, STATE_PAUSED, STATE_RUNNING, TIMER_TASKS_HISTORY_LOG
 
 
 apscheduler_logger = logging.getLogger('apscheduler')

scrapydweb/system/settings.py → scrapydweb/views/system/settings.py

@@ -5,9 +5,9 @@
 from flask import render_template
 from logparser import SETTINGS_PY_PATH as LOGPARSER_SETTINGS_PY_PATH
 
-from ..common import json_dumps
+from ...common import json_dumps
+from ...vars import SCHEDULER_STATE_DICT
 from ..myview import MyView
-from ..vars import SCHEDULER_STATE_DICT
 
 
 class SettingsView(MyView):