Skip to content

Commit b3b423d

Browse files
miguelgrinbergmiguelgrinberg
committed
dsl generator
1 parent f523369 commit b3b423d

File tree

7 files changed

+2035
-1
lines changed

7 files changed

+2035
-1
lines changed

‎noxfile.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,13 @@ def test_otel(session):
6767

6868
@nox.session()
6969
def format(session):
70-
session.install("black~=24.0", "isort", "flynt", "unasync>=0.6.0")
70+
session.install(
71+
"black~=24.0", "isort", "flynt", "unasync>=0.6.0", "jinja2", "elastic-transport"
72+
)
7173

7274
session.run("python", "utils/run-unasync.py")
7375
session.run("python", "utils/run-unasync-dsl.py")
76+
session.run("python", "utils/dsl-generator.py", env={"PYTHONPATH": "./"})
7477
session.run("isort", "--profile=black", *SOURCE_FILES)
7578
session.run("flynt", *SOURCE_FILES)
7679
session.run("black", *SOURCE_FILES)

‎utils/dsl-generator.py

Lines changed: 855 additions & 0 deletions
Large diffs are not rendered by default.

‎utils/run-unasync-dsl.py

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
# Licensed to Elasticsearch B.V. under one or more contributor
2+
# license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright
4+
# ownership. Elasticsearch B.V. licenses this file to you under
5+
# the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
import os
19+
import subprocess
20+
import sys
21+
from glob import glob
22+
from pathlib import Path
23+
24+
import unasync
25+
26+
27+
def main(check=False):
28+
# the list of directories that need to be processed with unasync
29+
# each entry has two paths:
30+
# - the source path with the async sources
31+
# - the destination path where the sync sources should be written
32+
source_dirs = [
33+
(
34+
"elasticsearch/dsl/_async/",
35+
"elasticsearch/dsl/_sync/",
36+
),
37+
("test_elasticsearch/test_dsl/_async/", "test_elasticsearch/test_dsl/_sync/"),
38+
(
39+
"test_elasticsearch/test_dsl/test_integration/_async/",
40+
"test_elasticsearch/test/dsl/test_integration/_sync/",
41+
),
42+
(
43+
"test_elasticsearch/test_dsl/test_integration/test_examples/_async/",
44+
"test_elasticsearch/test_dsl/test_integration/test_examples/_sync/",
45+
),
46+
("examples/dsl/async/", "examples/dsl/"),
47+
]
48+
49+
# Unasync all the generated async code
50+
additional_replacements = {
51+
"_async": "_sync",
52+
"AsyncElasticsearch": "Elasticsearch",
53+
"AsyncSearch": "Search",
54+
"AsyncMultiSearch": "MultiSearch",
55+
"AsyncEmptySearch": "EmptySearch",
56+
"AsyncDocument": "Document",
57+
"AsyncIndexMeta": "IndexMeta",
58+
"AsyncIndexTemplate": "IndexTemplate",
59+
"AsyncIndex": "Index",
60+
"AsyncComposableIndexTemplate": "ComposableIndexTemplate",
61+
"AsyncUpdateByQuery": "UpdateByQuery",
62+
"AsyncMapping": "Mapping",
63+
"AsyncFacetedSearch": "FacetedSearch",
64+
"AsyncUsingType": "UsingType",
65+
"async_connections": "connections",
66+
"async_scan": "scan",
67+
"async_simulate": "simulate",
68+
"async_bulk": "bulk",
69+
"async_mock_client": "mock_client",
70+
"async_client": "client",
71+
"async_data_client": "data_client",
72+
"async_write_client": "write_client",
73+
"async_pull_request": "pull_request",
74+
"async_examples": "examples",
75+
"async_sleep": "sleep",
76+
"assert_awaited_once_with": "assert_called_once_with",
77+
"pytest_asyncio": "pytest",
78+
"asynccontextmanager": "contextmanager",
79+
}
80+
rules = [
81+
unasync.Rule(
82+
fromdir=dir[0],
83+
todir=f"{dir[0]}_sync_check/" if check else dir[1],
84+
additional_replacements=additional_replacements,
85+
)
86+
for dir in source_dirs
87+
]
88+
89+
filepaths = []
90+
for root, _, filenames in os.walk(Path(__file__).absolute().parent.parent):
91+
if "/site-packages" in root or "/." in root or "__pycache__" in root:
92+
continue
93+
for filename in filenames:
94+
if filename.rpartition(".")[-1] in (
95+
"py",
96+
"pyi",
97+
) and not filename.startswith("utils.py"):
98+
filepaths.append(os.path.join(root, filename))
99+
100+
unasync.unasync_files(filepaths, rules)
101+
for dir in source_dirs:
102+
output_dir = f"{dir[0]}_sync_check/" if check else dir[1]
103+
subprocess.check_call(["black", "--target-version=py38", output_dir])
104+
subprocess.check_call(["isort", output_dir])
105+
for file in glob("*.py", root_dir=dir[0]):
106+
# remove asyncio from sync files
107+
subprocess.check_call(
108+
["sed", "-i.bak", "/^import asyncio$/d", f"{output_dir}{file}"]
109+
)
110+
subprocess.check_call(
111+
[
112+
"sed",
113+
"-i.bak",
114+
"s/asyncio\\.run(main())/main()/",
115+
f"{output_dir}{file}",
116+
]
117+
)
118+
subprocess.check_call(
119+
[
120+
"sed",
121+
"-i.bak",
122+
"s/elasticsearch\\[async\\]/elasticsearch/",
123+
f"{output_dir}{file}",
124+
]
125+
)
126+
subprocess.check_call(
127+
[
128+
"sed",
129+
"-i.bak",
130+
"s/pytest.mark.asyncio/pytest.mark.sync/",
131+
f"{output_dir}{file}",
132+
]
133+
)
134+
subprocess.check_call(["rm", f"{output_dir}{file}.bak"])
135+
136+
if check:
137+
# make sure there are no differences between _sync and _sync_check
138+
subprocess.check_call(
139+
[
140+
"diff",
141+
f"{dir[1]}{file}",
142+
f"{output_dir}{file}",
143+
]
144+
)
145+
146+
if check:
147+
subprocess.check_call(["rm", "-rf", output_dir])
148+
149+
150+
if __name__ == "__main__":
151+
main(check="--check" in sys.argv)

0 commit comments

Comments
 (0)