Skip to content

Fix ElasticSearch SQLClient deprecation warning #41871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 3 commits into from
Sep 1, 2024
Merged

Fix ElasticSearch SQLClient deprecation warning #41871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4726950
Remove importing SQLClient, and use sql interface directly from elast…
Owen-CH-Leung Aug 30, 2024
2a6c122
Fix failing CI test
Owen-CH-Leung Aug 30, 2024
b7d44d2
Remove checking pytest in modules, and patch ElasticSearch directly i…
Owen-CH-Leung Aug 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions airflow/providers/elasticsearch/hooks/elasticsearch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@

from deprecated import deprecated
from elasticsearch import Elasticsearch
from elasticsearch.client import SqlClient

from airflow.exceptions import AirflowProviderDeprecationWarning
from airflow.hooks.base import BaseHook
Expand Down Expand Up @@ -70,11 +69,10 @@ def __init__(
self.es = Elasticsearch(self.url, http_auth=(user, password), **self.kwargs)
else:
self.es = Elasticsearch(self.url, **self.kwargs)
self.es_sql_client = SqlClient(self.es)

def execute_sql(self, query: str) -> ObjectApiResponse:
sql_query = {"query": query}
return self.es_sql_client.query(body=sql_query)
return self.es.sql.query(body=sql_query)


class ElasticsearchSQLHook(DbApiHook):
Expand Down
11 changes: 7 additions & 4 deletions tests/providers/elasticsearch/hooks/test_elasticsearch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from __future__ import annotations

from unittest import mock
from unittest.mock import MagicMock

import pytest
from elasticsearch import Elasticsearch
Expand Down Expand Up @@ -119,16 +120,18 @@ def test_get_pandas_df(self):

self.cur.execute.assert_called_once_with(statement)

@mock.patch("airflow.providers.elasticsearch.hooks.elasticsearch.SqlClient.query")
def test_execute_sql_query(self, mock_query):
mock_query.return_value = {
@mock.patch("airflow.providers.elasticsearch.hooks.elasticsearch.Elasticsearch")
def test_execute_sql_query(self, mock_es):
mock_es_sql_client = MagicMock()
mock_es_sql_client.query.return_value = {
"columns": [{"name": "id"}, {"name": "first_name"}],
"rows": [[1, "John"], [2, "Jane"]],
}
mock_es.return_value.sql = mock_es_sql_client

es_connection = ESConnection(host="localhost", port=9200)
response = es_connection.execute_sql("SELECT * FROM index1")
mock_query.assert_called_once_with(body={"query": "SELECT * FROM index1"})
mock_es_sql_client.query.assert_called_once_with(body={"query": "SELECT * FROM index1"})

assert response["rows"] == [[1, "John"], [2, "Jane"]]
assert response["columns"] == [{"name": "id"}, {"name": "first_name"}]
Expand Down