Skip to content

Commit 9f6fc69

Browse files
merlimatmerlimat
authored
PIP-209: Compile Python client wrapper (#1)
* PIP-209: Compile Python client wrapper * Removed Optional * Fixed CMakefile * Added __pycache__ to gitignore * Added clang-format support
1 parent 0e325e6 commit 9f6fc69

File tree

12 files changed

+596
-38
lines changed

12 files changed

+596
-38
lines changed

‎.clang-format‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
19+
BasedOnStyle: Google
20+
IndentWidth: 4
21+
ColumnLimit: 110
22+
SortIncludes: false
23+
BreakBeforeBraces: Custom
24+
BraceWrapping:
25+
AfterEnum: true

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,10 @@ MANIFEST
22
build
33
dist
44
*.egg-info
5+
.idea
6+
CMakeCache.txt
7+
CMakeFiles
8+
Makefile
9+
_pulsar.so
10+
cmake_install.cmake
11+
__pycache__

‎CMakeLists.txt‎

Lines changed: 102 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,81 @@
1717
# under the License.
1818
#
1919

20-
INCLUDE_DIRECTORIES("${Boost_INCLUDE_DIRS}" "${PYTHON_INCLUDE_DIRS}")
20+
project (pulsar-client-python)
21+
cmake_minimum_required(VERSION 3.12)
22+
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
23+
24+
MESSAGE(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE})
25+
set(THREADS_PREFER_PTHREAD_FLAG TRUE)
26+
find_package(Threads REQUIRED)
27+
MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT})
28+
29+
30+
find_library(PULSAR_LIBRARY NAMES libpulsar.a)
31+
message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}")
32+
33+
find_path(PULSAR_INCLUDE pulsar/Client.h)
34+
message(STATUS "PULSAR_INCLUDE: ${PULSAR_INCLUDE}")
35+
36+
SET(Boost_NO_BOOST_CMAKE ON)
37+
SET(Boost_USE_STATIC_LIBS ON)
38+
39+
SET(CMAKE_CXX_STANDARD 11)
40+
41+
find_package(Boost)
42+
43+
find_package (Python3 COMPONENTS Development)
44+
MESSAGE(STATUS "PYTHON: " ${Python3_VERSION} " - " ${Python3_INCLUDE_DIRS})
45+
46+
string(REPLACE "." ";" PYTHONLIBS_VERSION_NO_LIST ${Python3_VERSION})
47+
48+
set(BOOST_PYTHON_NAME_POSTFIX ${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR})
49+
# For python3 the lib name is boost_python3
50+
set(BOOST_PYTHON_NAME_LIST python${BOOST_PYTHON_NAME_POSTFIX};python37;python38;python39;python310;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt)
51+
52+
foreach (BOOST_PYTHON_NAME IN LISTS BOOST_PYTHON_NAME_LIST)
53+
find_package(Boost QUIET COMPONENTS ${BOOST_PYTHON_NAME})
54+
if (${Boost_FOUND})
55+
set(BOOST_PYTHON_NAME_FOUND ${BOOST_PYTHON_NAME})
56+
break()
57+
endif()
58+
endforeach()
59+
60+
if (NOT ${Boost_FOUND})
61+
MESSAGE(FATAL_ERROR "Could not find Boost Python library")
62+
endif ()
63+
64+
MESSAGE(STATUS "BOOST_PYTHON_NAME_FOUND: " ${BOOST_PYTHON_NAME_FOUND})
65+
66+
set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/lib64/)
67+
68+
### This part is to find and keep SSL dynamic libs in RECORD_OPENSSL_SSL_LIBRARY and RECORD_OPENSSL_CRYPTO_LIBRARY
69+
### After find the libs, will unset related cache, and will not affect another same call to find_package.
70+
if (APPLE)
71+
set(OPENSSL_INCLUDE_DIR /usr/local/opt/openssl/include/ /opt/homebrew/opt/openssl/include)
72+
set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/local/opt/openssl/ /opt/homebrew/opt/openssl)
73+
endif ()
74+
75+
set(OPENSSL_USE_STATIC_LIBS TRUE)
76+
find_package(OpenSSL REQUIRED)
77+
78+
find_library(ZLIB_LIBRARIES REQUIRED NAMES libz.a z zlib)
79+
message(STATUS "ZLIB_LIBRARIES: ${ZLIB_LIBRARIES}")
80+
81+
find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a)
82+
message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}")
83+
find_library(Protobuf_LIBRARIES NAMES libprotobuf.a libprotobuf)
84+
message(STATUS "Protobuf: ${Protobuf_LIBRARIES}")
85+
find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a)
86+
message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}")
87+
find_library(LIB_ZSTD NAMES libzstd.a)
88+
message(STATUS "ZStd: ${LIB_ZSTD}")
89+
find_library(LIB_SNAPPY NAMES libsnappy.a)
90+
message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}")
91+
92+
########################################################################################################################
93+
94+
INCLUDE_DIRECTORIES(${PULSAR_INCLUDE} "${Boost_INCLUDE_DIRS}" "${Python3_INCLUDE_DIRS}")
2195

2296
ADD_LIBRARY(_pulsar SHARED src/pulsar.cc
2397
src/producer.cc
@@ -63,7 +137,14 @@ if (NOT DEFINED ${Boost_PYTHON310-MT_LIBRARY})
63137
endif()
64138

65139
# Try all possible boost-python variable namings
66-
set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY}
140+
set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY}
141+
${OPENSSL_LIBRARIES}
142+
${ZLIB_LIBRARIES}
143+
${CURL_LIBRARIES}
144+
${Protobuf_LIBRARIES}
145+
${LIB_ZSTD}
146+
${LIB_SNAPPY}
147+
${Boost_PYTHON_LIBRARY}
67148
${Boost_PYTHON3_LIBRARY}
68149
${Boost_PYTHON37-MT_LIBRARY}
69150
${Boost_PYTHON38_LIBRARY}
@@ -86,18 +167,33 @@ if (APPLE)
86167
endif ()
87168
endif()
88169

89-
message(STATUS "Using Boost Python libs: ${PYTHON_WRAPPER_LIBS}")
90-
91170
if (NOT PYTHON_WRAPPER_LIBS)
92171
MESSAGE(FATAL_ERROR "Could not find Boost Python library")
93172
endif ()
94173

174+
message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}")
175+
95176
if (APPLE)
96177
set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
97-
target_link_libraries(_pulsar -Wl,-all_load pulsarStatic ${PYTHON_WRAPPER_LIBS} ${COMMON_LIBS} ${ICU_LIBS})
178+
target_link_libraries(_pulsar -Wl,-all_load ${PYTHON_WRAPPER_LIBS})
98179
else ()
99180
if (NOT MSVC)
100181
set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc -static-libstdc++")
101182
endif()
102-
target_link_libraries(_pulsar pulsarStatic ${PYTHON_WRAPPER_LIBS} ${COMMON_LIBS})
183+
target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
103184
endif ()
185+
186+
find_package(ClangTools)
187+
set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
188+
add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py
189+
${CLANG_FORMAT_BIN}
190+
0
191+
${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
192+
${CMAKE_SOURCE_DIR}/src)
193+
194+
# `make check-format` option (for CI test)
195+
add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run_clang_format.py
196+
${CLANG_FORMAT_BIN}
197+
1
198+
${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
199+
${CMAKE_SOURCE_DIR}/src)

‎build-mac-wheels.sh‎

Lines changed: 44 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,9 @@ SNAPPY_VERSION=1.1.3
3939
CURL_VERSION=7.61.0
4040

4141
ROOT_DIR=$(git rev-parse --show-toplevel)
42-
cd "${ROOT_DIR}/pulsar-client-cpp"
42+
cd "${ROOT_DIR}"
4343

44+
PULSAR_VERSION=$(cat version.txt | grep pulsar-client-cpp | awk '{print $2}')
4445

4546
# Compile and cache dependencies
4647
CACHE_DIR=~/.pulsar-mac-wheels-cache
@@ -246,6 +247,43 @@ else
246247
echo "Using cached LibCurl"
247248
fi
248249

250+
###############################################################################
251+
if [ ! -f apache-pulsar-${PULSAR_VERSION}-src/.done ]; then
252+
echo "Building Pulsar C++ client - ${PULSAR_VERSION}"
253+
curl -O -L https://archive.apache.org/dist/pulsar/pulsar-${PULSAR_VERSION}/apache-pulsar-${PULSAR_VERSION}-src.tar.gz
254+
rm -rf apache-pulsar-${PULSAR_VERSION}-src/pulsar-client-cpp
255+
tar xfz apache-pulsar-${PULSAR_VERSION}-src.tar.gz
256+
pushd apache-pulsar-${PULSAR_VERSION}-src
257+
pushd pulsar-client-cpp
258+
ARCHS='arm64;x86_64'
259+
260+
chmod +x build-support/merge_archives.sh
261+
set -x
262+
cmake . \
263+
-DCMAKE_OSX_ARCHITECTURES=${ARCHS} \
264+
-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \
265+
-DCMAKE_INSTALL_PREFIX=$PREFIX \
266+
-DCMAKE_BUILD_TYPE=Release \
267+
-DCMAKE_PREFIX_PATH=$PREFIX \
268+
-DCMAKE_CXX_FLAGS=-I$PREFIX/include \
269+
-DBoost_INCLUDE_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/include \
270+
-DBoost_LIBRARY_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/lib \
271+
-DLINK_STATIC=OFF \
272+
-DBUILD_TESTS=OFF \
273+
-DBUILD_PYTHON_WRAPPER=OFF \
274+
-DBUILD_WIRESHARK=OFF \
275+
-DBUILD_DYNAMIC_LIB=OFF \
276+
-DBUILD_STATIC_LIB=ON \
277+
-DPROTOC_PATH=$PREFIX/bin/protoc
278+
279+
make -j16 install
280+
popd
281+
touch .done
282+
popd
283+
else
284+
echo "Using cached Pulsar C++ client"
285+
fi
286+
249287
###############################################################################
250288
###############################################################################
251289
###############################################################################
@@ -260,7 +298,7 @@ for line in "${PYTHON_VERSIONS[@]}"; do
260298
echo '----------------------------------------------------------------------------'
261299
echo "Build wheel for Python $PYTHON_VERSION"
262300

263-
cd "${ROOT_DIR}/pulsar-client-cpp"
301+
cd "${ROOT_DIR}"
264302

265303
find . -name CMakeCache.txt | xargs -r rm
266304
find . -name CMakeFiles | xargs -r rm -rf
@@ -285,16 +323,12 @@ for line in "${PYTHON_VERSIONS[@]}"; do
285323
-DCMAKE_CXX_FLAGS=-I$PREFIX/include \
286324
-DBoost_INCLUDE_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/include \
287325
-DBoost_LIBRARY_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/lib \
288-
-DPYTHON_INCLUDE_DIR=$PY_INCLUDE_DIR \
289-
-DPYTHON_LIBRARY=$PY_PREFIX/lib/libpython${PYTHON_VERSION}.dylib \
290-
-DLINK_STATIC=ON \
291-
-DBUILD_TESTS=OFF \
292-
-DBUILD_WIRESHARK=OFF \
293-
-DPROTOC_PATH=$PREFIX/bin/protoc
326+
-DPython3_INCLUDE_DIR=$PY_INCLUDE_DIR \
327+
-DPython3_LIBRARY=$PY_PREFIX/lib/libpython${PYTHON_VERSION}.dylib \
328+
-DPULSAR_INCLUDE=${PREFIX}/include
294329

295330
make clean
296-
make _pulsar -j16
331+
make -j16
297332

298-
cd python
299333
$PY_EXE setup.py bdist_wheel
300334
done

‎build-support/clang_format_exclusions.txt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#

‎build-support/run_clang_format.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/usr/bin/env python3
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
21+
# Original: https://github.com/apache/arrow/blob/4dbce607d50031a405af39d36e08cd03c5ffc764/cpp/build-support/run_clang_format.py
22+
# ChangeLog:
23+
# 2018-01-08: Accept multiple source directories (@Licht-T)
24+
25+
import fnmatch
26+
import os
27+
import subprocess
28+
import sys
29+
30+
if len(sys.argv) < 5:
31+
sys.stderr.write("Usage: %s $CLANG_FORMAT $CHECK_FORMAT exclude_globs.txt "
32+
"$source_dir1 $source_dir2\n" %
33+
sys.argv[0])
34+
sys.exit(1)
35+
36+
CLANG_FORMAT = sys.argv[1]
37+
CHECK_FORMAT = int(sys.argv[2]) == 1
38+
EXCLUDE_GLOBS_FILENAME = sys.argv[3]
39+
SOURCE_DIRS = sys.argv[4:]
40+
41+
exclude_globs = [line.strip() for line in open(EXCLUDE_GLOBS_FILENAME, "r")]
42+
43+
files_to_format = []
44+
matches = []
45+
for source_dir in SOURCE_DIRS:
46+
for directory, subdirs, files in os.walk(source_dir):
47+
for name in files:
48+
name = os.path.join(directory, name)
49+
if not (name.endswith('.h') or name.endswith('.cc')):
50+
continue
51+
52+
excluded = False
53+
for g in exclude_globs:
54+
if fnmatch.fnmatch(name, g):
55+
excluded = True
56+
break
57+
if not excluded:
58+
files_to_format.append(name)
59+
60+
if CHECK_FORMAT:
61+
output = subprocess.check_output([CLANG_FORMAT, '-output-replacements-xml']
62+
+ files_to_format,
63+
stderr=subprocess.STDOUT).decode('utf8')
64+
65+
to_fix = []
66+
for line in output.split('\n'):
67+
if 'offset' in line:
68+
to_fix.append(line)
69+
70+
if len(to_fix) > 0:
71+
print("clang-format checks failed, run 'make format' to fix")
72+
sys.exit(-1)
73+
else:
74+
try:
75+
cmd = [CLANG_FORMAT, '-i'] + files_to_format
76+
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
77+
except Exception as e:
78+
print(e)
79+
print(' '.join(cmd))
80+
raise

0 commit comments

Comments
 (0)