Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build: rework gyp files for zlib #33044

Closed
wants to merge 1 commit into from
Closed

build: rework gyp files for zlib #33044

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions deps/zlib/GN-scraper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Copyright (c) 2019 Refael Ackeramnn<refack@gmail.com>. All rights reserved.
# Use of this source code is governed by an MIT-style license.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to update/run license-builder.sh for this? And might this be better living in tools?

Copy link
Member Author

@richardlau richardlau Oct 17, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

¯\_(ツ)_/¯. It's a copy of the one in tools/v8_gypfiles/GN-scraper.py which we don't currently have in tools/license-builder.sh.

If I remember correctly I tried to reference the existing script/put the script in tools but couldn't get the gyp file to call it successfully via <!@pymod_do_main(.... Maybe someone else in @nodejs/gyp or @nodejs/python can advise.

import re
import os
import sys

PLAIN_SOURCE_RE = re.compile('\s*"([^/$].+)"\s*')
def DoMain(args):
gn_filename, pattern = args
src_root = os.path.dirname(gn_filename)
with open(gn_filename, 'rb') as gn_file:
gn_content = gn_file.read().decode('utf-8')

scraper_re = re.compile(pattern + r'\[([^\]]+)', re.DOTALL)
matches = scraper_re.search(gn_content)
match = matches.group(1)
files = []
for l in match.splitlines():
m2 = PLAIN_SOURCE_RE.match(l)
if not m2:
continue
files.append(m2.group(1))
# always use `/` since GYP will process paths further downstream
rel_files = ['"%s/%s"' % (src_root, f) for f in files]
return ' '.join(rel_files)

if __name__ == '__main__':
print(DoMain(sys.argv[1:]))
218 changes: 155 additions & 63 deletions deps/zlib/zlib.gyp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

{
'variables': {
'ZLIB_ROOT': '.',
'use_system_zlib%': 0,
'arm_fpu%': '',
'llvm_version%': '0.0',
Expand All @@ -12,44 +13,148 @@
['use_system_zlib==0', {
'targets': [
{
'target_name': 'zlib',
'target_name': 'zlib_adler32_simd',
'type': 'static_library',
'conditions': [
['(target_arch in "ia32 x64 x32" and OS!="ios") or arm_fpu=="neon"', {
'defines': [ 'ADLER32_SIMD_SSSE3' ],
'conditions': [
['OS=="win"', {
'defines': [ 'X86_WINDOWS' ],
},{
'defines': [ 'X86_NOT_WINDOWS' ],
}],
['OS!="win" or llvm_version!="0.0"', {
'cflags': [ '-mssse3' ],
}],
],
}],
['arm_fpu=="neon"', {
'defines': [ 'ADLER32_SIMD_NEON' ],
}],
],
'include_dirs': [ '<(ZLIB_ROOT)' ],
'defines': [ 'ZLIB_IMPLEMENTATION' ],
'direct_dependent_settings': {
'include_dirs': [ '<(ZLIB_ROOT)' ],
},
'sources': [
'<!@pymod_do_main(GN-scraper "<(ZLIB_ROOT)/BUILD.gn" "\\"zlib_adler32_simd\\".*?sources = ")',
],
}, # zlib_adler32_simd
{
'target_name': 'zlib_arm_crc32',
'type': 'static_library',
'conditions': [
['OS!="ios"', {
'conditions': [
['OS!="win" and llvm_version!="0.0"', {
'cflags': [ '-march=armv8-a+crc' ],
}],
['OS=="android"', {
'defines': [ 'ARMV8_OS_ANDROID' ],
}],
['OS=="linux"', {
'defines': [ 'ARMV8_OS_LINUX' ],
}],
['OS=="win"', {
'defines': [ 'ARMV8_OS_WINDOWS' ],
}],
],
'defines': [ 'CRC32_ARMV8_CRC32' ],
'include_dirs': [ '<(ZLIB_ROOT)' ],
'direct_dependent_settings': {
'include_dirs': [ '<(ZLIB_ROOT)' ],
},
'sources': [
'<!@pymod_do_main(GN-scraper "<(ZLIB_ROOT)/BUILD.gn" "\\"zlib_arm_crc32\\".*?sources = ")',
],
}],
],
'defines': [ 'ZLIB_IMPLEMENTATION' ],
}, # zlib_arm_crc32
{
'target_name': 'zlib_inflate_chunk_simd',
'type': 'static_library',
'conditions': [
['(target_arch in "ia32 x64 x32" and OS!="ios")', {
'defines': [ 'INFLATE_CHUNK_SIMD_SSE2' ],
'conditions': [
['target_arch=="x64"', {
'defines': [ 'INFLATE_CHUNK_READ_64LE' ],
}],
],
}],
['arm_fpu=="neon"', {
'defines': [ 'INFLATE_CHUNK_SIMD_NEON' ],
'conditions': [
['target_arch=="arm64"', {
'defines': [ 'INFLATE_CHUNK_READ_64LE' ],
}],
],
}]
],
'include_dirs': [ '<(ZLIB_ROOT)' ],
'defines': [ 'ZLIB_IMPLEMENTATION' ],
'direct_dependent_settings': {
'include_dirs': [ '<(ZLIB_ROOT)' ],
},
'sources': [
'<!@pymod_do_main(GN-scraper "<(ZLIB_ROOT)/BUILD.gn" "\\"zlib_inflate_chunk_simd\\".*?sources = ")',
],
}, # zlib_inflate_chunk_simd
{
'target_name': 'zlib_crc32_simd',
'type': 'static_library',
'conditions': [
['OS!="win" or llvm_version!="0.0"', {
'cflags': [
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this getting used at all? The latest test-macOS / test-macOS (pull_request) logs don't seem to include the cflags for this target in https://github.com/nodejs/node/pull/33044/checks?check_run_id=3008235129#step:5:1336:

  cc -o /Users/runner/work/node/node/out/Release/obj.target/zlib_crc32_simd/deps/zlib/crc32_simd.o ../deps/zlib/crc32_simd.c '-DV8_DEPRECATION_WARNINGS' '-DV8_IMMINENT_DEPRECATION_WARNINGS' '-D_GLIBCXX_USE_CXX11_ABI=1' '-D_DARWIN_USE_64_BIT_INODE=1' '-DOPENSSL_NO_PINSHARED' '-DOPENSSL_THREADS' '-DCRC32_SIMD_SSE42_PCLMUL' '-DZLIB_IMPLEMENTATION' -I../deps/zlib  -O3 -gdwarf-2 -mmacosx-version-min=10.13 -arch x86_64 -Wall -Wendif-labels -W -Wno-unused-parameter -fno-strict-aliasing -MMD -MF /Users/runner/work/node/node/out/Release/.deps//Users/runner/work/node/node/out/Release/obj.target/zlib_crc32_simd/deps/zlib/crc32_simd.o.d.raw   -c

'-msse4.2',
'-mpclmul',
],
}]
],
'defines': [
'CRC32_SIMD_SSE42_PCLMUL',
'ZLIB_IMPLEMENTATION',
],
'include_dirs': [ '<(ZLIB_ROOT)' ],
'direct_dependent_settings': {
'include_dirs': [ '<(ZLIB_ROOT)' ],
},
'sources': [
'adler32.c',
'compress.c',
'contrib/optimizations/insert_string.h',
'crc32.c',
'crc32.h',
'deflate.c',
'deflate.h',
'gzclose.c',
'gzguts.h',
'gzlib.c',
'gzread.c',
'gzwrite.c',
'infback.c',
'inffast.c',
'inffast.h',
'inffixed.h',
'inflate.h',
'inftrees.c',
'inftrees.h',
'trees.c',
'trees.h',
'uncompr.c',
'x86.h',
'zconf.h',
'zlib.h',
'zutil.c',
'zutil.h',
'<!@pymod_do_main(GN-scraper "<(ZLIB_ROOT)/BUILD.gn" "\\"zlib_crc32_simd\\".*?sources = ")',
],
'include_dirs': [
'.',
}, # zlib_crc32_simd
{
'target_name': 'zlib_x86_simd',
'type': 'static_library',
'conditions': [
['OS!="win" or llvm_version!="0.0"', {
'cflags': [
'-msse4.2',
'-mpclmul',
],
}]
],
'include_dirs': [ '<(ZLIB_ROOT)' ],
'defines': [ 'ZLIB_IMPLEMENTATION' ],
'direct_dependent_settings': {
'include_dirs': [
'.',
],
'include_dirs': [ '<(ZLIB_ROOT)' ],
},
'sources': [
'<!@pymod_do_main(GN-scraper "<(ZLIB_ROOT)/BUILD.gn" "\\"zlib_x86_simd\\".*?sources = ")',
],
}, # zlib_x86_simd
{
'target_name': 'zlib',
'type': 'static_library',
'sources': [
'<!@pymod_do_main(GN-scraper "<(ZLIB_ROOT)/BUILD.gn" "\\"zlib\\".*?sources = ")',
],
'include_dirs': [ '<(ZLIB_ROOT)' ],
'direct_dependent_settings': {
'include_dirs': [ '<(ZLIB_ROOT)' ],
},
'conditions': [
['OS!="win"', {
Expand All @@ -64,45 +169,31 @@
'USE_FILE32API'
],
}],
['(target_arch in "ia32 x64 x32" and OS!="ios") or arm_fpu=="neon"', {
'sources': [
'adler32_simd.c',
'adler32_simd.h',
'contrib/optimizations/chunkcopy.h',
'contrib/optimizations/inffast_chunk.c',
'contrib/optimizations/inffast_chunk.h',
'contrib/optimizations/inflate.c',
# Incorporate optimizations where possible.
# Optimizations not enabled for x86 as problems were reported,
# https://github.com/nodejs/node/issues/33019.
['(target_arch in "x64 x32" and OS!="ios") or arm_fpu=="neon"', {
'dependencies': [
'zlib_adler32_simd',
'zlib_inflate_chunk_simd',
],
}, {
'sources': [ 'inflate.c', ],
'defines': [ 'CPU_NO_SIMD' ],
'sources': [ '<(ZLIB_ROOT)/inflate.c', ],
}],
# Incorporate optimizations where possible
['target_arch in "ia32 x64 x32" and OS!="ios"', {
'defines': [
'ADLER32_SIMD_SSSE3',
'INFLATE_CHUNK_SIMD_SSE2',
'CRC32_SIMD_SSE42_PCLMUL',
'dependencies': [
'zlib_crc32_simd',
'zlib_x86_simd',
],
'sources': [
'crc32_simd.c',
'crc32_simd.h',
'crc_folding.c',
'fill_window_sse.c',
'x86.c',
],
'conditions': [
['target_arch=="x64"', {
'defines': [ 'INFLATE_CHUNK_READ_64LE' ],
}],
'<(ZLIB_ROOT)/x86.c',
],
}, {
'sources': [ 'simd_stub.c', ],
},{
'sources': [ '<(ZLIB_ROOT)/simd_stub.c' ],
}],
['arm_fpu=="neon"', {
'defines': [
'ADLER32_SIMD_NEON',
'INFLATE_CHUNK_SIMD_NEON',
],
'dependencies': [ 'zlib_arm_crc32' ],
'sources': [
'contrib/optimizations/slide_hash_neon.h',
],
Expand Down Expand Up @@ -138,6 +229,7 @@
],
}],
],
'defines': [ 'ZLIB_IMPLEMENTATION' ],
},
],
}, {
Expand Down
4 changes: 2 additions & 2 deletions doc/guides/maintaining-zlib.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ performance improvements not currently available in standard zlib.
Update zlib:
```bash
git clone https://chromium.googlesource.com/chromium/src/third_party/zlib
cp deps/zlib/zlib.gyp deps/zlib/win32/zlib.def deps
cp deps/zlib/zlib.gyp deps/zlib/GN-scraper.py deps/zlib/win32/zlib.def deps
rm -rf deps/zlib zlib/.git
mv zlib deps/
mv deps/zlib.gyp deps/zlib/
mv deps/zlib.gyp deps/GN-scraper.py deps/zlib/
mkdir deps/zlib/win32
mv deps/zlib.def deps/zlib/win32
sed -i -- 's_^#include "chromeconf.h"_//#include "chromeconf.h"_' deps/zlib/zconf.h
Expand Down