Skip to content

Commit 2359915

Browse files
timmusstimmuss
authored
Merge pull request #364 from znation/extract_dlls
Separate out mingw DLLs
2 parents 6429b09 + 54899ed commit 2359915

File tree

3 files changed

+149
-61
lines changed

3 files changed

+149
-61
lines changed

‎oss_local_scripts/make_egg.sh

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,14 @@ package_egg() {
233233
if [[ $OSTYPE == darwin* ]] || [[ $OSTYPE == msys ]]; then
234234
dist_type="bdist_wheel"
235235
fi
236+
237+
rm -rf sframe/libgcc_s_seh-1.dll \
238+
sframe/libstdc++-6.dll \
239+
sframe/cython/libgcc_s_seh-1.dll \
240+
sframe/cython/libstdc++-6.dll
241+
242+
push_ld_library_path
243+
236244
VERSION_NUMBER=`python -c "import sframe; print(sframe.version)"`
237245
${PYTHON_EXECUTABLE} setup.py ${dist_type} # This produced an egg/wheel starting with SFrame-${VERSION_NUMBER} under dist/
238246

@@ -281,6 +289,8 @@ package_egg() {
281289
unset PYTHONPATH
282290
$PYTHON_EXECUTABLE -c "import sframe; sframe.SArray(range(100)).apply(lambda x: x)"
283291

292+
pop_ld_library_path
293+
284294
# Done copy to the target directory
285295
cp $EGG_PATH ${TARGET_DIR}/
286296
echo -e "\n\n================= Done Packaging Egg ================\n\n"

‎oss_src/unity/python/sframe/__init__.py

Lines changed: 80 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -36,86 +36,105 @@
3636
from .connect import _get_metric_tracker
3737
from . import visualization
3838

39-
from .data_structures.sgraph import Vertex, Edge
40-
from .data_structures.sgraph import SGraph
41-
from .data_structures.sframe import SFrame
42-
from .data_structures.sarray import SArray
43-
from .data_structures.sketch import Sketch
44-
from .data_structures.image import Image
45-
from .data_structures.sarray_builder import SArrayBuilder
46-
from .data_structures.sframe_builder import SFrameBuilder
39+
import os as _os
40+
import sys as _sys
41+
if _sys.platform != 'win32' or \
42+
(_os.path.exists(_os.path.join(_os.path.dirname(__file__), 'cython', 'libstdc++-6.dll')) and \
43+
_os.path.exists(_os.path.join(_os.path.dirname(__file__), 'cython', 'libgcc_s_seh-1.dll'))):
44+
from .data_structures.sgraph import Vertex, Edge
45+
from .data_structures.sgraph import SGraph
46+
from .data_structures.sframe import SFrame
47+
from .data_structures.sarray import SArray
48+
from .data_structures.sketch import Sketch
49+
from .data_structures.image import Image
50+
from .data_structures.sarray_builder import SArrayBuilder
51+
from .data_structures.sframe_builder import SFrameBuilder
4752

48-
from .data_structures.sgraph import load_sgraph, load_graph
53+
from .data_structures.sgraph import load_sgraph, load_graph
4954

50-
from .toolkits._model import Model, CustomModel, load_model
55+
from .toolkits._model import Model, CustomModel, load_model
5156

52-
from . import aggregate
53-
from . import toolkits
57+
from . import aggregate
58+
from . import toolkits
5459

55-
from .toolkits.image_analysis import image_analysis
60+
from .toolkits.image_analysis import image_analysis
5661

57-
from .data_structures.sframe import load_sframe, get_spark_integration_jar_path
58-
from .data_structures.DBConnection import connect_odbc, get_libodbc_path, set_libodbc_path
62+
from .data_structures.sframe import load_sframe, get_spark_integration_jar_path
63+
from .data_structures.DBConnection import connect_odbc, get_libodbc_path, set_libodbc_path
5964

60-
# internal util
61-
from .connect.main import launch as _launch
62-
from .connect.main import stop as _stop
63-
from .connect import main as glconnect
65+
# internal util
66+
from .connect.main import launch as _launch
67+
from .connect.main import stop as _stop
68+
from .connect import main as glconnect
6469

6570

66-
from .util import get_environment_config
67-
from .util import get_graphlab_object_type
68-
from .util import get_log_location, get_client_log_location, get_server_log_location
71+
from .util import get_environment_config
72+
from .util import get_graphlab_object_type
73+
from .util import get_log_location, get_client_log_location, get_server_log_location
6974

70-
from .version_info import version
71-
from .version_info import __VERSION__
75+
from .version_info import version
76+
from .version_info import __VERSION__
7277

7378

74-
class DeprecationHelper(object):
75-
def __init__(self, new_target):
76-
self.new_target = new_target
79+
class DeprecationHelper(object):
80+
def __init__(self, new_target):
81+
self.new_target = new_target
7782

78-
def _warn(self):
79-
import warnings
80-
import logging
81-
warnings.warn("Graph has been renamed to SGraph. The Graph class will be removed in the next release.", PendingDeprecationWarning)
82-
logging.warning("Graph has been renamed to SGraph. The Graph class will be removed in the next release.")
83+
def _warn(self):
84+
import warnings
85+
import logging
86+
warnings.warn("Graph has been renamed to SGraph. The Graph class will be removed in the next release.", PendingDeprecationWarning)
87+
logging.warning("Graph has been renamed to SGraph. The Graph class will be removed in the next release.")
8388

84-
def __call__(self, *args, **kwargs):
85-
self._warn()
86-
return self.new_target(*args, **kwargs)
89+
def __call__(self, *args, **kwargs):
90+
self._warn()
91+
return self.new_target(*args, **kwargs)
8792

88-
def __getattr__(self, attr):
89-
self._warn()
90-
return getattr(self.new_target, attr)
93+
def __getattr__(self, attr):
94+
self._warn()
95+
return getattr(self.new_target, attr)
9196

92-
Graph = DeprecationHelper(SGraph)
97+
Graph = DeprecationHelper(SGraph)
9398

94-
from .cython import cy_pylambda_workers
95-
################### Extension Importing ########################
96-
from . import extensions
97-
from .extensions import ext_import
99+
from .cython import cy_pylambda_workers
100+
################### Extension Importing ########################
101+
from . import extensions
102+
from .extensions import ext_import
98103

99-
extensions._add_meta_path()
104+
extensions._add_meta_path()
100105

101-
# rewrite the extensions module
102-
class _extensions_wrapper(object):
103-
def __init__(self, wrapped):
104-
self._wrapped = wrapped
105-
self.__doc__ = wrapped.__doc__
106+
# rewrite the extensions module
107+
class _extensions_wrapper(object):
108+
def __init__(self, wrapped):
109+
self._wrapped = wrapped
110+
self.__doc__ = wrapped.__doc__
106111

107-
def __getattr__(self, name):
108-
try:
112+
def __getattr__(self, name):
113+
try:
114+
return getattr(self._wrapped, name)
115+
except:
116+
pass
117+
from .connect.main import get_unity
118+
get_unity()
109119
return getattr(self._wrapped, name)
110-
except:
111-
pass
112-
from .connect.main import get_unity
113-
get_unity()
114-
return getattr(self._wrapped, name)
115120

116-
import sys as _sys
117-
from . import _json as json # imports from _json.py in this directory
121+
from . import _json as json # imports from _json.py in this directory
122+
123+
_sys.modules[__name__ + ".extensions"] = _extensions_wrapper(_sys.modules[__name__ + ".extensions"])
124+
# rewrite the import
125+
extensions = _sys.modules[__name__ + ".extensions"]
126+
else:
127+
from dependencies import get_dependencies
128+
package_dir = _os.path.dirname(__file__)
129+
print("""
130+
ACTION REQUIRED: Dependencies libstdc++-6.dll and libgcc_s_seh-1.dll not found.
131+
132+
1. Ensure user account has write permission to %s
133+
2. Run sframe.get_dependencies() to download and install them.
134+
3. Restart Python and import sframe again.
135+
136+
By running the above function, you agree to the following licenses.
118137
119-
_sys.modules[__name__ + ".extensions"] = _extensions_wrapper(_sys.modules[__name__ + ".extensions"])
120-
# rewrite the import
121-
extensions = _sys.modules[__name__ + ".extensions"]
138+
* libstdc++: https://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
139+
* xz: http://git.tukaani.org/?p=xz.git;a=blob;f=COPYING
140+
""" % package_dir)

‎oss_src/unity/python/sframe/dependencies.py

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import os
2+
import shutil
3+
import subprocess
4+
import sys
5+
import tarfile
6+
import tempfile
7+
import urllib
8+
import zipfile
9+
10+
def get_dependencies():
11+
"""
12+
Downloads and installs external package dependencies.
13+
14+
1. Ensure user account has write permission to the install path of this package.
15+
2. Run sframe.get_dependencies() to download and install them.
16+
3. Restart Python and import sframe again.
17+
18+
By running the above function, you agree to the following licenses.
19+
20+
* libstdc++: https://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
21+
* xz: http://git.tukaani.org/?p=xz.git;a=blob;f=COPYING
22+
"""
23+
if sys.platform != 'win32':
24+
return
25+
print("""
26+
By running this function, you agree to the following licenses.
27+
28+
* libstdc++: https://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
29+
* xz: http://git.tukaani.org/?p=xz.git;a=blob;f=COPYING
30+
""")
31+
32+
print('Downloading xz.')
33+
(xzarchive_file, xzheaders) = urllib.urlretrieve('http://tukaani.org/xz/xz-5.2.1-windows.zip')
34+
xzarchive_dir = tempfile.mkdtemp()
35+
print('Extracting xz.')
36+
xzarchive = zipfile.ZipFile(xzarchive_file)
37+
xzarchive.extractall(xzarchive_dir)
38+
xz = os.path.join(xzarchive_dir, 'bin_x86-64', 'xz.exe')
39+
40+
print('Downloading gcc-libs.')
41+
(dllarchive_file, dllheaders) = urllib.urlretrieve('http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-libs-5.1.0-1-any.pkg.tar.xz')
42+
dllarchive_dir = tempfile.mkdtemp()
43+
44+
print('Extracting gcc-libs.')
45+
prev_cwd = os.getcwd()
46+
os.chdir(dllarchive_dir)
47+
subprocess.check_call([xz, '-d', dllarchive_file])
48+
dllarchive_tar = tarfile.open(os.path.splitext(dllarchive_file)[0])
49+
dllarchive_tar.extractall()
50+
51+
print('Copying gcc-libs into the installation directory.')
52+
package_dir = os.path.dirname(__file__)
53+
cython_dir = os.path.join(package_dir, 'cython')
54+
shutil.copyfile( \
55+
os.path.join('mingw64', 'bin', 'libgcc_s_seh-1.dll'), \
56+
os.path.join(cython_dir, 'libgcc_s_seh-1.dll'))
57+
shutil.copyfile( \
58+
os.path.join('mingw64', 'bin', 'libstdc++-6.dll'), \
59+
os.path.join(cython_dir, 'libstdc++-6.dll'))

0 commit comments

Comments
 (0)