Merge pull request #7801 from ydb-platform/mergelibs-240814-1820

Library import 240814-1820

Author: maximyurchuk

.piglet-meta.json

@@ -0,0 +1,3 @@
+{
+  "project":"ydblib"
+}

build/conf/java.conf

@@ -2066,3 +2066,14 @@ _KTLINT_BASELINE_FILE=
 macro KTLINT_BASELINE_FILE(File, Ticket) {
     SET(_KTLINT_BASELINE_FILE $File)
 }
+
+# tag:kotlin-specific
+_WITH_YA_1931=no
+
+# tag:kotlin-specific
+### @usage: WITH_YA_1931()
+### Interim macro to temporarily remove ALL_SRCDIRS from being added to ktlint test sources.
+macro WITH_YA_1931() {
+    MESSAGE(WARNING Some files in this project have style errors that were not caught by prior ktlint runs. Remove WITH_YA_1931 macro to see them.)
+    ENABLE(_WITH_YA_1931)
+}

build/conf/linkers/ld.conf

@@ -68,6 +68,13 @@ LD_ICF_FLAG+=$_LD_ICF_FLAG
 LD_ICF_FLAG+=$_LD_ICF_FLAG_PRINT_SECTIONS
 ICF_FLAG=
 
+_LD_NO_PIE_FLAG=
+when ($_LINKER_ID == "lld" && $PIE != "yes" && ($PIC != "yes" || $PIC_NO_PIE == "yes")) {
+    when ($OS_LINUX == "yes")  {
+        _LD_NO_PIE_FLAG=-Wl,-no-pie
+    }
+}
+
 STRIP_FLAG=
 
 C_LIBRARY_PATH=
@@ -159,6 +166,7 @@ _EXE_FLAGS=\
     $STRIP_FLAG \
     $DCE_FLAG \
     $ICF_FLAG \
+    $_LD_NO_PIE_FLAG \
     $_LINKER_TIME_TRACE_FLAG
 
 _SONAME_FLAG=-Wl,$_SONAME_OPTION,$_SONAME

build/conf/opensource.conf

@@ -54,21 +54,32 @@ when ($OPENSOURCE == "yes" && $EXPORT_CMAKE == "yes") {
     EXPORTED_BUILD_SYSTEM_BUILD_ROOT="${PROJECT_BINARY_DIR}"
 }
 
-CMAKE_PACKAGE=
-CMAKE_PACKAGE_COMPONENT=
-CMAKE_PACKAGE_TARGET=
-CONAN_REFERENCE=
-CONAN_PKG_OPTS=
-### @usage: OPENSOURCE_EXPORT_REPLACEMENT(CMAKE PkgName CMAKE_TARGET PkgName::PkgTarget CONAN ConanRef CMAKE_COMPONENT OptCmakePkgComponent)
-###
-### Use specified conan/system pacakcge when exporting cmake build scripts for arcadia C++ project
-### for opensource publication.
-macro OPENSOURCE_EXPORT_REPLACEMENT(CMAKE[], CMAKE_TARGET[], CMAKE_COMPONENT[], CONAN[], CONAN_OPTIONS[]) {
-    SET(CMAKE_PACKAGE $CMAKE)
-    SET(CMAKE_PACKAGE_COMPONENT $CMAKE_COMPONENT)
+CMAKE_PACKAGE_SEM=
+CONAN_REQUIRES_SEM=
+CONAN_OPTIONS_SEM=
+CONAN_OS_REQUIRES_SEM=
+CONAN_OS_OPTIONS_SEM=
+macro _OPENSOURCE_EXPORT_REPLACEMENT_CMAKE(CMAKE[], CMAKE_TARGET[], CMAKE_COMPONENT[]) {
+    SET(CMAKE_PACKAGE_SEM packages-ITEM && packages-name $CMAKE ${pre= && packages-components :CMAKE_COMPONENT} && mpackages-ITEM && mpackages-name $CMAKE ${pre= && mpackages-components :CMAKE_COMPONENT} && find_package $CMAKE ${pre= COMPONENTS :CMAKE_COMPONENT} )
     SET(CMAKE_LINK_TARGET $CMAKE_TARGET)
-    SET(CONAN_REFERENCE $CONAN)
-    SET(CONAN_PKG_OPTS $CONAN_OPTIONS)
+}
+
+### @usage: OPENSOURCE_EXPORT_REPLACEMENT(CMAKE PkgName CMAKE_COMPONENT OptCmakePkgComponent CMAKE_TARGET PkgName::PkgTarget CONAN ConanRequire CONAN ConanOptions CONAN_ADDITIONAL_SEMS ConanAdditionalSems)
+###
+### Use specified conan/system package when exporting cmake build scripts for arcadia C++ project for opensource publication.
+macro OPENSOURCE_EXPORT_REPLACEMENT(CMAKE[], CMAKE_TARGET[], CMAKE_COMPONENT[], CONAN[], CONAN_OPTIONS[], CONAN_ADDITIONAL_SEMS[]) {
+    _OPENSOURCE_EXPORT_REPLACEMENT_CMAKE(CMAKE $CMAKE CMAKE_TARGET $CMAKE_TARGET CMAKE_COMPONENT $CMAKE_COMPONENT)
+    SET(CONAN_REQUIRES_SEM && conan_require $CONAN && conan-requires $CONAN $CONAN_ADDITIONAL_SEMS)
+    SET(CONAN_OPTIONS_SEM ${pre=&& conan_options :CONAN_OPTIONS} ${pre=&& conan-options :CONAN_OPTIONS})
+}
+
+### @usage: OPENSOURCE_EXPORT_REPLACEMENT_BY_OS(OS Os CMAKE PkgName CMAKE_COMPONENT OptCmakePkgComponent CMAKE_TARGET PkgName::PkgTarget CONAN ConanRequire CONAN ConanOptions CONAN_ADDITIONAL_SEMS ConanAdditionalSems)
+###
+### Use specified conan/system package when exporting cmake build scripts for arcadia C++ project for opensource publication.
+macro OPENSOURCE_EXPORT_REPLACEMENT_BY_OS(OS[], CMAKE[], CMAKE_TARGET[], CMAKE_COMPONENT[], CONAN[], CONAN_OPTIONS[], CONAN_ADDITIONAL_SEMS[]) {
+    _OPENSOURCE_EXPORT_REPLACEMENT_CMAKE(CMAKE $CMAKE CMAKE_TARGET $CMAKE_TARGET CMAKE_COMPONENT $CMAKE_COMPONENT)
+    SET(CONAN_OS_REQUIRES_SEM && conan-os_depends-ITEM && conan-os_depends-os $OS && conan-os_depends-requires $CONAN $CONAN_ADDITIONAL_SEMS)
+    SET(CONAN_OS_OPTIONS_SEM && conan-os_depends-ITEM && conan-os_depends-os $OS ${pre=&& conan-os_depends-opts :CONAN_OPTIONS})
 }
 
 CMAKE_TARGET_NAME=$REALPRJNAME

build/conf/settings.conf

@@ -14,10 +14,25 @@ FORCE_RESOLVE_MACRO_INCLS=yes
 ENABLE_RERESOLVE_FOR_GENERATED_FILES=yes
 REPORT_ALL_DUPSRC=yes
 DEPS_CACHE_CONTROL_UIDS_CACHE=yes
+INCLUDE_BLACKLIST_TO_CONF_HASH=no
 INCLUDE_ISOLATED_PROJECTS_TO_CONF_HASH=yes
+USE_GLOBAL_CMD=yes
+YMAKE_USE_NEW_UIDS=yes
+USE_GRAPH_CHANGES_PREDICTOR=yes
+PIC_NO_PIE=no
+FAIL_PY2=no
 
-when ($YMAKE_USE_OLD_UIDS != "yes") {
-    YMAKE_USE_NEW_UIDS=yes
+when ($YMAKE_USE_OLD_UIDS == "yes") {
+    YMAKE_USE_NEW_UIDS=no
+}
+
+when ($OS_WINDOWS == "yes") {
+    USE_GLOBAL_CMD=no
+}
+
+when ($AUTOCHECK == "yes") {
+    USE_GRAPH_CHANGES_PREDICTOR=no
+    PIC_NO_PIE=yes
 }
 
 # NOTE! Only foldable vars should be listed here
@@ -110,3 +125,12 @@ _FOLDABLE_VARS=\
     _STD_CXX_VERSION \
 
 # end of _FOLDABLE_VARS list
+
+ARCADIA_TEST_ROOT=../arcadia_tests_data/
+DEFAULT_REQUIREMENTS=network:restricted cpu:1 ram:32
+
+__COMMA__=${comma:""}
+__BSDQ__=\"
+when ($STRUCT_CMD == "yes") {
+    __BSDQ__=\\\"
+}

build/conf/ts/node_modules.conf

@@ -7,13 +7,13 @@ NPM_SCRIPT=$NPM_ROOT/node_modules/npm/bin/npm-cli.js
 PM_SCRIPT=
 PM_TYPE=
 
-NPM_CONTRIBS_PATH=contrib/typescript
+NPM_CONTRIBS_PATH=-
 # combined input/outputs records as list of directives ${input;hide:<path>} ${output;hide:<path>}, used in builders
 _NODE_MODULES_INOUTS=
 _YATOOL_PREBUILDER_ARG=
 
 macro CUSTOM_CONTRIB_TYPESCRIPT(P) {
-    SET(NPM_CONTRIBS_PATH $P)
+    SET(NPM_CONTRIBS_PATH -)
 }
 
 macro NO_CONTRIB_TYPESCRIPT() {

build/export_generators/cmake/conan_imports.jinja

@@ -0,0 +1,13 @@
+{%- for conan_import in conan_imports -%}
+{%-     set search_and_dstdir = rsplit(conan_import|trim, "->", 2) -%}
+{%-     set search = search_and_dstdir[0]|trim -%}
+{%-     set dstdir = search_and_dstdir[1]|trim -%}
+{%-     set srcdir_and_mask = rsplit(search, ",", 2) -%}
+{%-     set srcdir = srcdir_and_mask[0]|trim -%}
+{%-     set mask = srcdir_and_mask[1]|trim -%}
+{%-     if use_conan2 %}
+                {{ shift }}copy(self, pattern="{{ mask }}", src=dep.cpp_info.bindirs[0], dst=self.build_folder + "../../../../{{ dstdir }}")
+{%-     else %}
+        {{ shift }}self.copy(pattern="{{ mask }}", src="{{ srcdir }}", dst="{{ dstdir }}")
+{%-     endif -%}
+{%- endfor -%}

build/export_generators/cmake/conanfile.py.jinja

@@ -19,15 +19,51 @@ class App(ConanFile):
 {%- endfor -%}
 }
 
-    requires =
-{%- for conan_require in conan.requires %} "{{ conan_require }}"
-{%-     if not loop.last -%},{%- endif -%}
-{%- endfor %}
-
-    tool_requires =
-{%- for conan_tool_require in conan.tool_requires %} "{{ conan_tool_require }}"
-{%-     if not loop.last -%},{%- endif -%}
-{%- endfor %}
+{%- set has_conan_os_depends_requires = conan.os_depends|selectattr('requires')|map(attribute='requires')|sum|length -%}
+{%- if (conan.requires|length) or (has_conan_os_depends_requires) %}
+
+    def requirements(self):
+{%-     if (conan.requires|length) %}
+{%-         for conan_require in conan.requires %}
+        self.requires("{{ conan_require }}")
+{%-         endfor -%}
+{%-     endif -%}
+
+{%-     if (has_conan_os_depends_requires) -%}
+{%-         for conan_os_depend in conan.os_depends|selectattr('requires') %}
+{%-             if (conan_os_depend.requires|length) %}
+        if self.settings.os == "{{ conan_os_depend.os }}":
+{%-                 for conan_require in conan_os_depend.requires %}
+            self.requires("{{ conan_require }}")
+{%-                 endfor -%}
+{%-             endif -%}
+{%-         endfor -%}
+{%-     endif -%}
+{%- endif -%}
+
+{%- set has_conan_os_depends_tool_requires = conan.os_depends|selectattr('tool_requires')|map(attribute='tool_requires')|sum|length -%}
+{%- if (conan.tool_requires|length) or (has_conan_os_depends_tool_requires) %}
+
+    def build_requirements(self):
+{%-     if (conan.tool_requires|length) %}
+{%-         for conan_tool_require in conan.tool_requires %}
+        self.tool_requires("{{ conan_tool_require }}")
+{%-         endfor -%}
+{%-     endif -%}
+
+{%-     if (has_conan_os_depends_tool_requires) -%}
+{%-         for conan_os_depend in conan.os_depends|selectattr('tool_requires') %}
+{%-             if (conan_os_depend.tool_requires|length) %}
+        if self.settings.os == "{{ conan_os_depend.os }}":
+{%-                 for conan_tool_require in conan_os_depend.tool_requires %}
+            self.tool_requires("{{ conan_tool_require }}")
+{%-                 endfor -%}
+{%-             endif -%}
+{%-         endfor -%}
+{%-     endif -%}
+{%- endif -%}
+
+{%- set has_conan_os_depends_imports = conan.os_depends|selectattr('imports')|map(attribute='imports')|sum|length -%}
 
 {%- if use_conan2 %}
 
@@ -40,32 +76,47 @@ class App(ConanFile):
 
         for dep in self.dependencies.values():
             if dep.cpp_info.bindirs:
-{%-         for conan_import in conan.imports -%}
-{%-             set search_and_dstdir = rsplit(conan_import|trim, "->", 2) -%}
-{%-             set search = search_and_dstdir[0]|trim -%}
-{%-             set dstdir = search_and_dstdir[1]|trim -%}
-{%-             set srcdir_and_mask = rsplit(search, ",", 2) -%}
-{%-             set srcdir = srcdir_and_mask[0]|trim -%}
-{%-             set mask = srcdir_and_mask[1]|trim %}
-                copy(self, pattern="{{ mask }}", src=dep.cpp_info.bindirs[0], dst=self.build_folder + "../../../../{{ dstdir }}")
-{%-         endfor %}
+{%-         if conan.imports|length -%}
+{%-             set conan_imports = conan.imports -%}
+{%-             set shift = "" -%}
+{%-             include "[generator]/conan_imports.jinja" -%}
+{%-         endif -%}
+{%-         if (has_conan_os_depends_imports) -%}
+{%-             for conan_os_depend in conan.os_depends|selectattr('imports') %}
+{%-                 if (conan_os_depend.imports|length) %}
+                if self.settings.os == "{{ conan_os_depend.os }}":
+{%-                     set conan_imports = conan_os_depend.imports -%}
+{%-                     set shift = "    " -%}
+{%-                     include "[generator]/conan_imports.jinja" -%}
+{%-                 endif -%}
+{%-             endfor -%}
+{%-         endif -%}
 
     def layout(self):
         cmake_layout(self)
 
-{%- else -%}
+{%- else %}
 
     generators = "cmake_find_package", "cmake_paths"
 
     def imports(self):
-{%-     for conan_import in conan.imports -%}
-{%-         set search_and_dstdir = rsplit(conan_import|trim, "->", 2) -%}
-{%-         set search = search_and_dstdir[0]|trim -%}
-{%-         set dstdir = search_and_dstdir[1]|trim -%}
-{%-         set srcdir_and_mask = rsplit(search, ",", 2) -%}
-{%-         set srcdir = srcdir_and_mask[0]|trim -%}
-{%-         set mask = srcdir_and_mask[1]|trim %}
-        self.copy(pattern="{{ mask }}", src="{{ srcdir }}", dst="{{ dstdir }}")
-{%-     endfor %}
-
-{%- endif %}
+{%- if conan.imports|length -%}
+{%-     set conan_imports = conan.imports -%}
+{%-     set shift = "" -%}
+{%-     include "[generator]/conan_imports.jinja" -%}
+{%- endif -%}
+
+{%- if (has_conan_os_depends_imports) -%}
+{%-     for conan_os_depend in conan.os_depends|selectattr('imports') %}
+{%-         if (conan_os_depend.imports|length) %}
+        if self.settings.os == "{{ conan_os_depend.os }}":
+{%-             set conan_imports = conan_os_depend.imports -%}
+{%-             set shift = "    " -%}
+{%-             include "[generator]/conan_imports.jinja" -%}
+{%-         endif -%}
+{%-     endfor -%}
+{%- endif -%}
+
+{%- endif -%}
+
+{%- include "[generator]/debug_conan.jinja" ignore missing %}

build/export_generators/cmake/generator.toml

@@ -87,12 +87,23 @@ include="skip"
 
 # All Conan attributes make sorted_set for stability generated conanfile,
 # order has no effect for all conan options
+# All OS conan attributes
 conan-requires="sorted_set"
 conan-tool_requires="sorted_set"
 conan-imports="sorted_set"
 conan-options="sorted_set"
 conan-generators="sorted_set"
 
+# OS depends conan attributes
+conan-os_depends="list"
+conan-os_depends-ITEM="dict"
+conan-os_depends-os="str"
+conan-os_depends-requires="sorted_set"
+conan-os_depends-tool_requires="sorted_set"
+conan-os_depends-imports="sorted_set"
+conan-os_depends-options="sorted_set"
+conan-os_depends-generators="sorted_set"
+
 # For project languages order important only for ASM, it must be after C/CXX,
 # this special logic maked in template, here use sorted_set for stability generated vars
 project_languages="sorted_set"

build/export_generators/hardcoded-cmake/conanfile.py.jinja

@@ -19,15 +19,27 @@ class App(ConanFile):
 {%- endfor -%}
 }
 
-    requires =
-{%- for conan_require in conan_packages %} "{{ conan_require }}"
-{%-     if not loop.last -%},{%- endif -%}
-{%- endfor %}
+{%- if (conan_tool_packages|length) %}
 
     tool_requires =
-{%- for conan_tool_require in conan_tool_packages %} "{{ conan_tool_require }}"
-{%-     if not loop.last -%},{%- endif -%}
-{%- endfor %}
+{%-     for conan_tool_require in conan_tool_packages %} "{{ conan_tool_require }}"
+{%-         if not loop.last -%},{%- endif -%}
+{%-     endfor -%}
+{%- endif -%}
+
+{%- if (conan_packages|length) %}
+
+    def requirements(self):
+        requires = [
+{%-     for conan_require in conan_packages %} "{{ conan_require }}"
+{%-         if not loop.last -%},{%- endif -%}
+{%-     endfor %} ]
+        for require in requires:
+            if ("linux-headers" in require) and (self.settings.os != "Linux"):
+                continue
+            self.requires(require)
+
+{%- endif -%}
 
 {%- if use_conan2 %}
 
@@ -50,7 +62,7 @@ class App(ConanFile):
     def layout(self):
         cmake_layout(self)
 
-{%- else -%}
+{%- else %}
 
     generators = "cmake_find_package", "cmake_paths"