added instrumentation for mysql-connector and pymysql (#603)

* added instrumentation for mysql-connector-python

* added support for instrumenting pymysql

Author: beniwohli

.ci/.jenkins_framework.yml

@@ -30,3 +30,5 @@ FRAMEWORK:
   - eventlet-newest
   - gevent-newest
   - zerorpc-0.4
+  - mysql_connector-newest
+  - pymysql-newest

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
     -   id: black
         language_version: python3
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v1.3.0
+    rev: v2.3.0
     hooks:
     -   id: flake8
         exclude: elasticapm\/utils\/wrapt|build|src|tests|dist|conftest.py|setup.py

CHANGELOG.md

@@ -5,11 +5,10 @@
 
 ### New Features
 
+ * added instrumentation for mysql-connector and pymysql (#603)
  * implemented stack_trace_limit configuration option (#623)
  * autoinsert tracing middleware in django settings (#625)
 
-### 
-
 ## v5.2.3
 [Check the diff](https://github.com/elastic/apm-agent-python/compare/v5.2.2...v5.2.3)
 

docs/supported-technologies.asciidoc

@@ -91,7 +91,7 @@ Collected trace data:
 
 [float]
 [[automatic-instrumentation-db-mysql]]
-==== MySQL
+==== MySQLdb
 
 Library: `MySQLdb`
 
@@ -101,6 +101,38 @@ Instrumented methods:
 
 The instrumented `connect` method returns a wrapped connection/cursor which instruments the actual `Cursor.execute` calls.
 
+Collected trace data:
+
+ * parametrized SQL query
+
+[float]
+[[automatic-instrumentation-db-mysql-connector]]
+==== mysql-connector
+
+Library: `mysql-connector-python`
+
+Instrumented methods:
+
+ * `mysql.connector.connect`
+ 
+The instrumented `connect` method returns a wrapped connection/cursor which instruments the actual `Cursor.execute` calls.
+
+Collected trace data:
+
+ * parametrized SQL query
+
+[float]
+[[automatic-instrumentation-db-pymysql]]
+==== pymysql
+
+Library: `pymysql`
+
+Instrumented methods:
+
+ * `pymysql.connect`
+ 
+The instrumented `connect` method returns a wrapped connection/cursor which instruments the actual `Cursor.execute` calls.
+
 Collected trace data:
 
  * parametrized SQL query

elasticapm/instrumentation/packages/mysql_connector.py

@@ -0,0 +1,57 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+from elasticapm.instrumentation.packages.dbapi2 import (
+    ConnectionProxy,
+    CursorProxy,
+    DbApi2Instrumentation,
+    extract_signature,
+)
+
+
+class MySQLCursorProxy(CursorProxy):
+    provider_name = "mysql"
+
+    def extract_signature(self, sql):
+        return extract_signature(sql)
+
+
+class MySQLConnectionProxy(ConnectionProxy):
+    cursor_proxy = MySQLCursorProxy
+
+
+class MySQLConnectorInstrumentation(DbApi2Instrumentation):
+    name = "mysql_connector"
+
+    instrument_list = [("mysql.connector", "connect")]
+
+    def call(self, module, method, wrapped, instance, args, kwargs):
+        return MySQLConnectionProxy(wrapped(*args, **kwargs))

elasticapm/instrumentation/packages/pymysql.py

@@ -0,0 +1,56 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from elasticapm.instrumentation.packages.dbapi2 import (
+    ConnectionProxy,
+    CursorProxy,
+    DbApi2Instrumentation,
+    extract_signature,
+)
+
+
+class PyMySQLCursorProxy(CursorProxy):
+    provider_name = "mysql"
+
+    def extract_signature(self, sql):
+        return extract_signature(sql)
+
+
+class PyMySQLConnectionProxy(ConnectionProxy):
+    cursor_proxy = PyMySQLCursorProxy
+
+
+class PyMySQLConnectorInstrumentation(DbApi2Instrumentation):
+    name = "pymysql"
+
+    instrument_list = [("pymysql", "connect")]
+
+    def call(self, module, method, wrapped, instance, args, kwargs):
+        return PyMySQLConnectionProxy(wrapped(*args, **kwargs))

elasticapm/instrumentation/register.py

@@ -37,6 +37,8 @@
     "elasticapm.instrumentation.packages.psycopg2.Psycopg2Instrumentation",
     "elasticapm.instrumentation.packages.psycopg2.Psycopg2ExtensionsInstrumentation",
     "elasticapm.instrumentation.packages.mysql.MySQLInstrumentation",
+    "elasticapm.instrumentation.packages.mysql_connector.MySQLConnectorInstrumentation",
+    "elasticapm.instrumentation.packages.pymysql.PyMySQLConnectorInstrumentation",
     "elasticapm.instrumentation.packages.pylibmc.PyLibMcInstrumentation",
     "elasticapm.instrumentation.packages.pymongo.PyMongoInstrumentation",
     "elasticapm.instrumentation.packages.pymongo.PyMongoBulkInstrumentation",

setup.cfg

@@ -28,6 +28,9 @@ markers =
     memcached
     redis
     psutil
+    mysql_connector
+    pymysql
+    mysqldb
 
 [isort]
 line_length=120

tests/docker-compose.yml

@@ -124,6 +124,17 @@ services:
     volumes:
       - pymssqldata:/var/opt/mssql
 
+  mysql:
+    image: mysql
+    command: --default-authentication-plugin=mysql_native_password --log_error_verbosity=3
+    environment:
+      - MYSQL_DATABASE=eapm_tests
+      - MYSQL_USER=eapm
+      - MYSQL_PASSWORD=Very(!)Secure
+      - MYSQL_RANDOM_ROOT_PASSWORD=yes
+    volumes:
+      - mysqldata:/var/lib/mysql
+
   run_tests:
     image: apm-agent-python:${PYTHON_VERSION}
     environment:
@@ -156,3 +167,5 @@ volumes:
     driver: local
   pymssqldata:
     driver: local
+  mysqldata:
+    driver: local

tests/instrumentation/mysql_connector_tests.py

@@ -0,0 +1,86 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+
+import pytest
+
+from elasticapm.conf.constants import TRANSACTION
+
+connector = pytest.importorskip("mysql.connector")
+
+pytestmark = [pytest.mark.mysql_connector]
+
+
+if "MYSQL_HOST" not in os.environ:
+    pytestmark.append(pytest.mark.skip("Skipping mysql-connector tests, no MYSQL_HOST environment variable set"))
+
+
+@pytest.yield_fixture(scope="function")
+def mysql_connector_connection(request):
+    conn = connector.connect(
+        host=os.environ.get("MYSQL_HOST", "localhost"),
+        user=os.environ.get("MYSQL_USER", "eapm"),
+        password=os.environ.get("MYSQL_PASSWORD", ""),
+        database=os.environ.get("MYSQL_DATABASE", "eapm_tests"),
+    )
+    cursor = conn.cursor()
+    cursor.execute("CREATE TABLE `test` (`id` INT, `name` VARCHAR(5))")
+    cursor.execute("INSERT INTO `test` (`id`, `name`) VALUES (1, 'one'), (2, 'two'), (3, 'three')")
+    row = cursor.fetchone()
+    print(row)
+
+    yield conn
+
+    cursor.execute("DROP TABLE `test`")
+
+
+@pytest.mark.integrationtest
+def test_mysql_connector_select(instrument, mysql_connector_connection, elasticapm_client):
+    cursor = mysql_connector_connection.cursor()
+    query = "SELECT * FROM test WHERE name LIKE 't%' ORDER BY id"
+
+    try:
+        elasticapm_client.begin_transaction("web.django")
+        cursor.execute(query)
+        assert cursor.fetchall() == [(2, "two"), (3, "three")]
+        elasticapm_client.end_transaction(None, "test-transaction")
+    finally:
+        transactions = elasticapm_client.events[TRANSACTION]
+        spans = elasticapm_client.spans_for_transaction(transactions[0])
+        span = spans[0]
+        assert span["name"] == "SELECT FROM test"
+        assert span["type"] == "db"
+        assert span["subtype"] == "mysql"
+        assert span["action"] == "query"
+        assert "db" in span["context"]
+        assert span["context"]["db"]["type"] == "sql"
+        assert span["context"]["db"]["statement"] == query