Update deps, fix tox for pypy, update pypy results

Author: grigi

README.rst

@@ -207,59 +207,60 @@ Geometric Mean     6686.61    6392.17   17593.94       11127.67    6191.21     1
 =============== ========== ========== ========== ============== ========== ============
 
 
-PyPy7.2-Py3.6:
+PyPy7.3-Py3.6:
 
 =============== ========== ========== ========== ============== ============
 Test 1          Django     peewee     Pony ORM   SQLAlchemy ORM Tortoise ORM
 =============== ========== ========== ========== ============== ============
-Insert: Single     4844.65    5699.46    6069.36         928.29      4119.27
-Insert: Batch      6623.80    5942.15   14888.52        6750.90     19889.84
-Insert: Bulk      15833.01   19876.12          —       19456.28     59792.55
-Filter: Large    145843.64   96627.96  212065.79      110034.60     83070.61
-Filter: Small      4867.78   61654.60  172243.99       56155.40     40043.36
-Get                3538.23    7448.17    5911.03        4175.23      8464.85
-Filter: dict     142764.62  109864.27  131992.03      121185.63     94067.34
-Filter: tuple    143723.62  106352.47  206592.66      263009.74    126491.99
-Update: Whole      6448.56   14572.05   19806.92       23459.62     25520.65
-Update: Partial    7320.68   20191.26   32069.98       40098.96     36326.02
-Delete             8583.36   29682.59   73035.62       83364.88     37541.98
-Geometric Mean    15281.28   25564.53   43575.11       27864.52     33369.34
+Insert: Single     4092.94    6042.45    6166.89        1004.98      6786.58
+Insert: Batch      4529.93    6456.81   18247.22        6982.63     26348.64
+Insert: Bulk      17961.11   24302.27          —       21428.22     80531.38
+Filter: Large    152801.52   91886.54  295678.67      129700.40     90993.88
+Filter: Small      6099.36   65094.77  175134.68       60966.19     45463.05
+Get                4255.07    6793.41    8310.16        4339.15      9229.52
+Filter: dict     147533.08  116293.38  215108.01      109211.59     94985.63
+Filter: tuple    175529.83  122951.45  281181.48      253415.27    130914.54
+Update: Whole      6710.01   16514.91   41939.12       22677.70     30434.61
+Update: Partial    8089.69   23377.04   51308.13       43023.59     38576.48
+Delete             8766.41   29169.88   74799.44       81948.65     42805.28
+Geometric Mean    15887.12   27270.66   58524.96       28825.51     39281.41
 =============== ========== ========== ========== ============== ============
 
 =============== ========== ========== ========== ============== ============
 Test 2          Django     peewee     Pony ORM   SQLAlchemy ORM Tortoise ORM
 =============== ========== ========== ========== ============== ============
-Insert: Single     4856.53    6099.71    6302.97         886.47      4709.30
-Insert: Batch      6388.57    5979.01   14255.97        4636.63     19616.08
-Insert: Bulk      15675.79   18751.54          —       17444.69     62711.06
-Filter: Large    130201.72   86533.17  311024.40      110243.82     69154.04
-Filter: Small      3845.09   65312.52  183754.86       47955.16     36348.92
-Get                3540.29    7934.10    9121.36        4233.04      6100.74
-Filter: dict     138333.47  105259.77  237840.16      119919.30     77601.55
-Filter: tuple    155187.72   89280.79  335520.30      260503.32    116994.87
-Update: Whole      6693.48   13079.63   37170.20       18749.98     22658.09
-Update: Partial    7609.05   17583.81   47285.79       27172.58     35797.50
-Delete             2339.78   36752.54   41859.66         655.36     40024.90
-Geometric Mean    13247.47   25048.86    55454.4       15927.71     31084.92
+Insert: Single     4089.62    5982.16    5927.49         818.31      8128.96
+Insert: Batch      4582.76    6909.47   15558.25        6012.19     25381.23
+Insert: Bulk      16201.10   24021.67          —       20294.09     77993.66
+Filter: Large    138968.39   90818.94  279382.51      118860.29     71640.16
+Filter: Small      5439.62   62951.57  168192.03       52251.13     38208.34
+Get                4092.11    6989.34    8230.02        3379.57      8430.82
+Filter: dict     134900.00  112626.68  202932.98       94477.51     71689.52
+Filter: tuple    159685.66  122797.29  274293.13      223882.76    119104.10
+Update: Whole      6201.26   11396.24   35644.86       17562.70     28303.72
+Update: Partial    7669.88   23086.17   41247.77       25492.40     35430.58
+Delete             2087.76   34330.64   38098.81         633.66       369.97
+Geometric Mean    13135.78   26719.72   50653.72       15519.07     23445.05
 =============== ========== ========== ========== ============== ============
 
 =============== ========== ========== ========== ============== ============
 Test 3          Django     peewee     Pony ORM   SQLAlchemy ORM Tortoise ORM
 =============== ========== ========== ========== ============== ============
-Insert: Single     1965.02    3519.71    4207.43         670.29      3592.01
-Insert: Batch      1904.72    3493.31    7636.38        4421.69     12099.74
-Insert: Bulk       4054.34    7156.34          —       13786.11     24692.28
-Filter: Large     20301.33   29613.15  122365.02       38721.68      2855.62
-Filter: Small      1908.01   26244.67  105406.30       24031.37      2478.21
-Get                2111.95    4936.50    6633.01        2315.14      1348.12
-Filter: dict      28415.33   39443.07   75871.80       28089.40      2995.77
-Filter: tuple     26695.73   39080.83  125314.67       58677.38     10422.97
-Update: Whole      4072.91    5559.29   27090.75       11341.40     12903.72
-Update: Partial    7782.08   20601.30   37512.08       14907.51     31801.49
-Delete             9303.80   26078.32   59424.56       63702.66     42910.38
-Geometric Mean     5855.94   12832.73   33099.23       12839.01      7698.50
+Insert: Single     2194.07    3827.50    4030.12         792.54      5429.88
+Insert: Batch      2072.86    3928.59    7509.87        4841.25     15489.04
+Insert: Bulk       4747.82    9996.01          —       15407.30     29085.53
+Filter: Large     25016.73   30627.76  122459.86       37727.96      2968.21
+Filter: Small      1508.74   24123.13   98162.15       21523.32      2454.28
+Get                2231.56    4443.86    6313.33        2312.70      1490.23
+Filter: dict      29467.52   40064.73   81433.44       27085.70      3001.12
+Filter: tuple     31329.65   46774.06  123617.06       45894.78      8845.51
+Update: Whole      4220.60    6984.34   29109.60       10686.28     11302.96
+Update: Partial    7346.76   21125.93   33835.74       14716.48     24182.52
+Delete             9083.28   31221.47   64601.85       64029.41     41709.27
+Geometric Mean     6146.34   14064.74   32867.64       12702.84      7951.76
 =============== ========== ========== ========== ============== ============
 
+
 Results (PostgreSQL)
 ====================
 
@@ -374,6 +375,60 @@ Geometric Mean      5549.9    3979.97   12006.83        7386.97    4155.09
 =============== ========== ========== ========== ============== ========== ============ ================
 
 
+PyPy7.3-Py3.6:
+
+=============== ========== ========== ========== ============== ============
+Test 1          Django     peewee     Pony ORM   SQLAlchemy ORM Tortoise ORM
+=============== ========== ========== ========== ============== ============
+Insert: Single     2479.55    2663.10    3088.68         686.92      3311.91
+Insert: Batch      3478.12    4571.76    5194.87        4214.03     13584.52
+Insert: Bulk      14553.90   19480.48          —       15260.41     55214.98
+Filter: Large     80983.35  175029.85  479457.80       59215.04    160185.46
+Filter: Small      4995.72   37628.16  136060.23       18990.66     24888.53
+Get                2868.25    4870.54    6107.97        2630.39      6538.67
+Filter: dict      80650.88  219339.95  301358.89       52242.19    183104.50
+Filter: tuple     93584.59  257332.26  490594.29       74740.17    175407.06
+Update: Whole      3563.43    7760.36    5348.80        5540.23     11161.77
+Update: Partial    4536.02   10036.94   11210.07       12264.97     14984.03
+Delete             4978.72   10073.98   11107.52       10907.25     12449.24
+Geometric Mean     9889.22   20926.09   30192.66       11285.31     26393.92
+=============== ========== ========== ========== ============== ============
+
+=============== ========== ========== ========== ============== ============
+Test 2          Django     peewee     Pony ORM   SQLAlchemy ORM Tortoise ORM
+=============== ========== ========== ========== ============== ============
+Insert: Single     2266.32    2663.82    2669.34         597.25      3513.88
+Insert: Batch      3328.18    4435.96    6949.97        3773.89     10493.08
+Insert: Bulk      14065.42   18684.71          —       14458.93     55861.34
+Filter: Large     80296.63  164763.64  447302.97       54498.39    153077.80
+Filter: Small      4800.15   35434.65  130211.62       17627.65     21258.96
+Get                2565.44    4543.19    5849.48        2310.20      6251.27
+Filter: dict      77842.51  207108.12  280970.71       50958.86    185933.49
+Filter: tuple     91267.58  239574.53  438762.04       73630.97    172285.95
+Update: Whole      2701.75    5406.82    6975.43        4327.97      9913.26
+Update: Partial    4539.54    9879.36   10746.90       10125.79     14495.50
+Delete             1176.44    7017.19    6249.98        1387.88      8181.26
+Geometric Mean     8165.06   18884.27   28591.56        8489.41     24079.39
+=============== ========== ========== ========== ============== ============
+
+=============== ========== ========== ========== ============== ============
+Test 3          Django     peewee     Pony ORM   SQLAlchemy ORM Tortoise ORM
+=============== ========== ========== ========== ============== ============
+Insert: Single     1008.08    1582.11    1617.94         465.03      2258.82
+Insert: Batch      1492.21    2833.39    3595.88        2126.38      6739.82
+Insert: Bulk       3357.71    6484.93          —        6410.32     16790.12
+Filter: Large     12164.83   40576.68  109275.41        9531.62     34746.82
+Filter: Small      2314.81   12551.29   58031.65        5503.53      9873.51
+Get                1083.01    2165.71    2800.69         854.14      2358.51
+Filter: dict      12742.82   54209.15   73098.84        8640.37     46526.40
+Filter: tuple     13728.33   62009.01  107794.63       10255.39     42013.15
+Update: Whole      1589.21    2352.14    4388.50        3946.67      5021.95
+Update: Partial    3894.78    8822.50    6142.39        7243.80     14487.33
+Delete             3791.23    8238.25    8413.57        8394.18     11540.60
+Geometric Mean      3367.9    8574.74   13385.55         4134.8     11175.14
+=============== ========== ========== ========== ============== ============
+
+
 Quick analysis
 --------------
 * Pony ORM is heavily optimised for performance.
@@ -382,14 +437,20 @@ Quick analysis
 * ``Get`` is surprisingly slow for everyone.
 * Pony ORM, SQLAlchemy & SQLObject does merge operations for updates, so is technically always partial updates.
 * Tortoise ORM performance using the ``asyncpg`` PostgreSQL driver is really good, winning overall.
-* Tortoise ORM performance using the ``aiomysql`` MySQL driver is mediocre, the driver itself is taking the majority of CPU time.
+* Tortoise ORM performance using the ``aiomysql`` MySQL driver is mediocre, the driver itself is taking the majority of CPU time. PyPy runs this driver a lot faster, which indicates that the slow paths are likely just Python itself.
 
-PyPy comparison
----------------
+PyPy comparison: SQLite
+-----------------------
 * ``peewee`` and ``Pony ORM`` gets a noticeable performance improvement
-* ``SQLAlchemy ORM`` is marginally faster
-* ``Django`` is marginally slower
-* ``Tortoise ORM`` is notably slower, but notably faster for Updates & Deletes
+* ``SQLAlchemy ORM`` and ``Django`` performs similarily
+* ``Tortoise ORM`` has slow Reads and fast Create, Update & Delete operation
+* ``SQLObject`` fails
+
+PyPy comparison: MySQL
+-----------------------
+* ``peewee`` and ``Tortoise ORM`` gets a noticeable performance improvement
+* ``Pony ORM`` is marginally faster
+* ``SQLAlchemy ORM`` and ``Django`` is marginally slower
 * ``SQLObject`` fails
 
 

src/bench.sh

@@ -4,6 +4,9 @@ export ITERATIONS=200
 if [ "x$1" == "xfull" ]; then
     export ITERATIONS=2000
 fi
+if [ "x$1" == "xextra" ]; then
+    export ITERATIONS=10000
+fi
 
 cd $(dirname $0)
 

src/bench_all.sh

@@ -3,7 +3,7 @@
 echo > results
 
 DBTYPE=sqlite ./bench.sh $1
-DBTYPE=postgres ./bench.sh $1
+python -V | grep PyPy || DBTYPE=postgres ./bench.sh $1
 DBTYPE=mysql ./bench.sh $1
 
 cat results

src/requirements.in

@@ -1,21 +1,21 @@
 # ORM's
-django<3
-django_jsonfield  # Requires django<3
+django
+django_jsonfield
 sqlalchemy
 peewee
 pony
 sqlobject
 tortoise-orm
 
 # DB drivers
-asyncpg
+asyncpg ; implementation_name == "cpython"
 aiomysql
 mysqlclient
 psycopg2
 
 # Recommended accelerators
-uvloop
-python-rapidjson
+uvloop ; sys_platform != "win32" and implementation_name == "cpython"
+python-rapidjson ; implementation_name == "cpython"
 
 # Testing tools
 vmprof

src/requirements.txt

@@ -5,47 +5,48 @@
 #    pip-compile
 #
 aiomysql==0.0.20
-aiosqlite==0.10.0         # via tortoise-orm
-asyncpg==0.20.0
+aiosqlite==0.11.0         # via tortoise-orm
+asgiref==3.2.3            # via django
+asyncpg==0.20.0 ; implementation_name == "cpython"
 certifi==2019.11.28       # via requests
 cffi==1.13.2              # via cryptography
 chardet==3.0.4            # via requests
 ciso8601==2.1.2           # via tortoise-orm
 colorama==0.4.3           # via vmprof
 cryptography==2.8         # via pymysql
-django-jsonfield==1.3.1
-django==2.2.8
+django-jsonfield==1.4.0
+django==3.0.2
 filelock==3.0.12          # via tox
 formencode==1.3.1         # via sqlobject
 idna==2.8                 # via requests
-importlib-metadata==1.2.0  # via pluggy, tox
+importlib-metadata==1.3.0  # via pluggy, tox
 more-itertools==8.0.2     # via zipp
 mysqlclient==1.4.6
-packaging==19.2           # via tox
+packaging==20.0           # via tox
 peewee==3.13.1
 pluggy==0.13.1            # via tox
 pony==0.7.11
 psycopg2==2.8.4
-py-spy==0.3.1
-py==1.8.0                 # via tox
+py-spy==0.3.2
+py==1.8.1                 # via tox
 pycparser==2.19           # via cffi
 pydispatcher==2.0.5       # via sqlobject
 pymysql==0.9.2            # via aiomysql
-pyparsing==2.4.5          # via packaging
-pypika==0.35.16           # via tortoise-orm
-python-rapidjson==0.9.1
+pyparsing==2.4.6          # via packaging
+pypika==0.35.18           # via tortoise-orm
+python-rapidjson==0.9.1 ; implementation_name == "cpython"
 pytz==2019.3              # via django, vmprof
 requests==2.22.0          # via vmprof
-six==1.13.0               # via cryptography, packaging, tox, vmprof
-sqlalchemy==1.3.11
+six==1.13.0               # via cryptography, django-jsonfield, packaging, tox, vmprof
+sqlalchemy==1.3.12
 sqlobject==3.8.0
 sqlparse==0.3.0           # via django
 toml==0.10.0              # via tox
-tortoise-orm==0.15.4
-tox==3.14.2
+tortoise-orm==0.15.7
+tox==3.14.3
 typing-extensions==3.7.4.1  # via tortoise-orm
 urllib3==1.25.7           # via requests
-uvloop==0.14.0
-virtualenv==16.7.8        # via tox
+uvloop==0.14.0 ; sys_platform != "win32" and implementation_name == "cpython"
+virtualenv==16.7.9        # via tox
 vmprof==0.4.15
 zipp==0.6.0               # via importlib-metadata

src/tortoise/bench.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 import os
+import sys
 try:
-    import os
     concurrents = int(os.environ.get('CONCURRENTS', '1'))
 
     if concurrents > 1:
@@ -16,6 +16,9 @@
 finally:
     pass
 
+if concurrents > 1 and sys.version_info < (3,7):
+    sys.exit()
+
 dbtype = os.environ.get('DBTYPE', '')
 if dbtype == 'postgres':
     db_url = f'postgres://postgres:@127.0.0.1:5432/tbench?minsize={concurrents}&maxsize={concurrents}'

src/tox.ini

@@ -4,7 +4,11 @@ skip_missing_interpreters = True
 
 [testenv]
 commands=
-    ./bench.sh full
+    ./bench_all.sh full
 deps =
     -r requirements.txt
 
+[testenv:pypy3]
+commands=
+    ./bench_all.sh extra
+