Performance tuning (#51)

grigi · abondar · commit 201083335924 · 2018-09-28T11:51:35.000+03:00
* __init__ restructure give 25-30% speedup in select benchmark.

I also cleared up a few lint warnings.

* Fix check

* v0.10.8

* Fix bad refactor

* Allow NPE to pass through
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,5 +1,10 @@
 Changelog
 =========
+
+0.10.8
+------
+- Performance fixes from ``pypika`` and object retrieval
+
 0.10.7
 ------
 - Fixed SQLite relative db path and :memory: now also works
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -19,7 +19,7 @@ certifi==2018.8.24        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
 ciso8601==2.0.1
-click==6.7                # via pip-tools
+click==7.0                # via pip-tools
 cloud-sptheme==1.9.4
 colorama==0.3.9           # via green
 coverage==4.5.1           # via coveralls, green
@@ -41,7 +41,7 @@ markupsafe==1.0           # via jinja2
 mccabe==0.6.1             # via flake8, pylint
 mypy-extensions==0.4.1    # via mypy
 mypy==0.630
-packaging==17.1           # via sphinx
+packaging==18.0           # via sphinx
 pbr==4.2.0                # via stevedore
 pip-tools==3.0.0
 pluggy==0.7.1             # via tox
@@ -53,7 +53,7 @@ pygments==2.2.0
 pylint==2.1.1
 pymysql==0.9.2            # via aiomysql
 pyparsing==2.2.1          # via packaging
-pypika==0.15.5
+pypika==0.15.6
 pytz==2018.5              # via babel
 pyyaml==3.13              # via bandit
 requests==2.19.1          # via coveralls, sphinx
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
-pypika>=0.15.4,<1.0
+pypika>=0.15.6,<1.0
 ciso8601>=2.0
 aiocontextvars==0.1.2;python_version<"3.7"
 aiosqlite>=0.6.0
diff --git a/tortoise/__init__.py b/tortoise/__init__.py
@@ -374,4 +374,4 @@ async def do_stuff():
         loop.run_until_complete(Tortoise.close_connections())
 
 
-__version__ = "0.10.7"
+__version__ = "0.10.8"
diff --git a/tortoise/backends/asyncpg/client.py b/tortoise/backends/asyncpg/client.py
@@ -18,8 +18,8 @@ class AsyncpgDBClient(BaseDBAsyncClient):
     executor_class = AsyncpgExecutor
     schema_generator = AsyncpgSchemaGenerator
 
-    def __init__(self, user, password, database, host, port, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+    def __init__(self, user, password, database, host, port, **kwargs):
+        super().__init__(**kwargs)
 
         self.user = user
         self.password = password
@@ -107,7 +107,7 @@ def _in_transaction(self):
         else:
             return self._transaction_class(self.connection_name, pool=self._db_pool)
 
-    async def execute_query(self, query):
+    async def execute_query(self, query, get_inserted_id=False):
         try:
             async with self.acquire_connection() as connection:
                 self.log.debug(query)
@@ -153,6 +153,7 @@ def __init__(self, connection_name, pool=None, connection=None):
         self._transaction_class = self.__class__
         self._old_context_value = None
         self.connection_name = connection_name
+        self.transaction = None
 
     def acquire_connection(self):
         return ConnectionWrapper(self._connection)
diff --git a/tortoise/backends/base/client.py b/tortoise/backends/base/client.py
@@ -37,7 +37,7 @@ def acquire_connection(self):
     def _in_transaction(self):
         raise NotImplementedError()  # pragma: nocoverage
 
-    async def execute_query(self, query):
+    async def execute_query(self, query, get_inserted_id=False):
         raise NotImplementedError()  # pragma: nocoverage
 
     async def execute_script(self, script):
@@ -62,10 +62,12 @@ async def __aexit__(self, exc_type, exc_val, exc_tb):
 
 
 class SingleConnectionWrapper(BaseDBAsyncClient):
+    # pylint: disable=W0223,W0231
+
     def __init__(self, connection_name, connection, closing_callback=None):
+        self.log = logging.getLogger('db_client')
         self.connection_name = connection_name
         self.connection = connection
-        self.log = logging.getLogger('db_client')
         self.single_connection = True
         self.closing_callback = closing_callback
 
diff --git a/tortoise/backends/base/config_generator.py b/tortoise/backends/base/config_generator.py
@@ -10,7 +10,7 @@
 DB_LOOKUP = {
     'postgres': {
         'engine': 'tortoise.backends.asyncpg',
-        'vars': {
+        'vmap': {
             'path': 'database',
             'hostname': 'host',
             'port': 'port',
@@ -21,13 +21,13 @@
     'sqlite': {
         'engine': 'tortoise.backends.sqlite',
         'skip_first_char': False,
-        'vars': {
+        'vmap': {
             'path': 'file_path',
         },
     },
     'mysql': {
         'engine': 'tortoise.backends.mysql',
-        'vars': {
+        'vmap': {
             'path': 'database',
             'hostname': 'host',
             'port': 'port',
@@ -61,20 +61,20 @@ def expand_db_url(db_url: str, testing: bool = False) -> dict:
         path = path.replace('\\{', '{').replace('\\}', '}')
         path = path.format(uuid.uuid4().hex)
 
-    vars = {}  # type: dict
-    vars.update(db['vars'])
-    params[vars['path']] = path
-    if vars.get('hostname'):
-        params[vars['hostname']] = str(url.hostname or '')
+    vmap = {}  # type: dict
+    vmap.update(db['vmap'])
+    params[vmap['path']] = path
+    if vmap.get('hostname'):
+        params[vmap['hostname']] = str(url.hostname or '')
     try:
-        if vars.get('port'):
-            params[vars['port']] = str(url.port or '')
+        if vmap.get('port'):
+            params[vmap['port']] = str(url.port or '')
     except ValueError:
         raise ConfigurationError('Port is not an integer')
-    if vars.get('username'):
-        params[vars['username']] = str(url.username or '')
-    if vars.get('password'):
-        params[vars['password']] = str(url.password or '')
+    if vmap.get('username'):
+        params[vmap['username']] = str(url.username or '')
+    if vmap.get('password'):
+        params[vmap['password']] = str(url.password or '')
 
     return {
         'engine': db['engine'],
diff --git a/tortoise/backends/base/schema_generator.py b/tortoise/backends/base/schema_generator.py
@@ -47,9 +47,6 @@ def _get_primary_key_create_string(self, field_name):
         # has to implement in children
         raise NotImplementedError()  # pragma: nocoverage
 
-    def _get_auto_now_add_default(self):
-        raise NotImplementedError()  # pragma: nocoverage
-
     def _get_table_sql(self, model) -> dict:
         fields_to_create = []
         m2m_tables_for_create = []
diff --git a/tortoise/backends/mysql/client.py b/tortoise/backends/mysql/client.py
@@ -18,8 +18,8 @@ class MySQLClient(BaseDBAsyncClient):
     executor_class = MySQLExecutor
     schema_generator = MySQLSchemaGenerator
 
-    def __init__(self, user, password, database, host, port, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+    def __init__(self, user, password, database, host, port, **kwargs):
+        super().__init__(**kwargs)
 
         self.user = user
         self.password = password
diff --git a/tortoise/backends/sqlite/client.py b/tortoise/backends/sqlite/client.py
@@ -16,8 +16,8 @@ class SqliteClient(BaseDBAsyncClient):
     executor_class = SqliteExecutor
     schema_generator = SqliteSchemaGenerator
 
-    def __init__(self, file_path, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+    def __init__(self, file_path, **kwargs):
+        super().__init__(**kwargs)
         self.filename = file_path
         self._transaction_class = type(
             'TransactionWrapper', (TransactionWrapper, self.__class__), {}
diff --git a/tortoise/fields.py b/tortoise/fields.py
@@ -29,7 +29,7 @@ class Field:
     """
     def __init__(
         self,
-        type=None,
+        type=None,  # pylint: disable=W0622
         source_field: Optional[str] = None,
         generated: bool = False,
         pk: bool = False,
@@ -301,7 +301,7 @@ def __init__(
 
 
 class BackwardFKRelation:
-    def __init__(self, type, relation_field, **kwargs):
+    def __init__(self, type, relation_field, **kwargs):  # pylint: disable=W0622
         self.type = type
         self.relation_field = relation_field
 
diff --git a/tortoise/models.py b/tortoise/models.py
@@ -328,21 +328,17 @@ class Model(metaclass=ModelMeta):
 
     def __init__(self, *args, **kwargs) -> None:
         is_new = not bool(kwargs.get('id'))
+        passed_fields = set(kwargs.keys())
 
-        for key, field in self._meta.fields_map.items():
-            if isinstance(field, fields.BackwardFKRelation):
-                setattr(
-                    self, key,
-                    RelationQueryContainer(field.type, field.relation_field, self, is_new)
-                )
-            elif isinstance(field, fields.ManyToManyField):
-                setattr(self, key, ManyToManyRelationManager(field.type, self, field, is_new))
-            elif isinstance(field, fields.Field):
-                setattr(self, key, field.default)
-            else:
-                setattr(self, key, None)
+        for key in self._meta.backward_fk_fields:
+            field = self._meta.fields_map[key]
+            setattr(self, key, RelationQueryContainer(
+                field.type, field.relation_field, self, is_new))  # type: ignore
+
+        for key in self._meta.m2m_fields:
+            field = self._meta.fields_map[key]
+            setattr(self, key, ManyToManyRelationManager(field.type, self, field, is_new))
 
-        passed_fields = set(kwargs.keys())
         for key, value in kwargs.items():
             if key in self._meta.fk_fields:
                 if hasattr(value, 'id') and not value.id:
@@ -351,6 +347,13 @@ def __init__(self, *args, **kwargs) -> None:
                 relation_field = '{}_id'.format(key)
                 setattr(self, relation_field, value.id)
                 passed_fields.add(relation_field)
+            elif key in self._meta.fields:
+                field_object = self._meta.fields_map[key]
+                if value is None and not field_object.null:
+                    raise ValueError('{} is non nullable field, but null was passed'.format(key))
+                setattr(self, key, field_object.to_python_value(value))
+            elif key in self._meta.db_fields:
+                setattr(self, self._meta.fields_db_projection_reverse[key], value)
             elif key in self._meta.backward_fk_fields:
                 raise ConfigurationError(
                     'You can\'t set backward relations through init, change related model instead'
@@ -359,18 +362,11 @@ def __init__(self, *args, **kwargs) -> None:
                 raise ConfigurationError(
                     'You can\'t set m2m relations through init, use m2m_manager instead'
                 )
-            elif key in self._meta.fields:
-                field_object = self._meta.fields_map[key]
-                if value is None and not field_object.null:
-                    raise ValueError('{} is non nullable field, but null was passed'.format(key))
-                setattr(self, key, field_object.to_python_value(value))
-            elif key in self._meta.db_fields:
-                setattr(self, self._meta.fields_db_projection_reverse[key], value)
+
+        passed_fields.update(self._meta.fetch_fields)
 
         for key, field_object in self._meta.fields_map.items():
-            if key in passed_fields or key in self._meta.fetch_fields:
-                continue
-            else:
+            if key not in passed_fields:
                 if callable(field_object.default):
                     setattr(self, key, field_object.default())
                 else:
diff --git a/tortoise/queryset.py b/tortoise/queryset.py
@@ -284,7 +284,7 @@ def annotate(self, **kwargs) -> 'QuerySet':
             queryset._available_custom_filters.update(get_filters_for_field(key, None, key))
         return queryset
 
-    def values_list(self, *fields: str, flat: bool = False):
+    def values_list(self, *fields: str, flat: bool = False):  # pylint: disable=W0621
         """
         Make QuerySet returns list of tuples for given args instead of objects.
         If ```flat=True`` and only one arg is passed can return flat list.
@@ -581,6 +581,8 @@ async def _execute(self):
 
 
 class FieldSelectQuery(AwaitableQuery):
+    # pylint: disable=W0223
+
     def _join_table_with_forwarded_fields(self, model, field, forwarded_fields):
         table = Table(model._meta.table)
         if field in model._meta.fields_db_projection and not forwarded_fields:

Original file line number	Diff line number	Diff line change
`@@ -374,4 +374,4 @@ async def do_stuff():`
`374`	`374`	`loop.run_until_complete(Tortoise.close_connections())`
`375`	`375`
`376`	`376`
`377`		`-__version__ = "0.10.7"`
	`377`	`+__version__ = "0.10.8"`