Merge branch 'rightlib' into merge-libs-250117-0020

Author: alexv-smirnov

build/conf/autoincludes.json

@@ -1,4 +1,5 @@
 [
+    "devtools/ya",
     "library/cpp/geo",
     "util"
 ]

build/conf/java.conf

@@ -831,15 +831,18 @@ MAVEN_EXPORT_CMD=
 JAVA_COVERAGE_SRCLIST_FLAG=--coverage ${output;pre=${MODULE_PREFIX};suf=.cpsf:REALPRJNAME} --source-root ${ARCADIA_ROOT}
 JAVA_COVERAGE_SRCLIST=
 PREPARE_JAVA_BUILD_DIRS=$FS_TOOLS md ${BINDIR}/cls && $FS_TOOLS md ${BINDIR}/misc
-EXTRACT_GENDIRS=${cwd:BINDIR} $YMAKE_PYTHON ${input:"build/scripts/autotar_gendirs.py"} --unpack --ext .gentar ${ext=.gentar:AUTO_INPUT}
-COLLECT_JAVA_SRCLIST=${YMAKE_PYTHON} ${input:"build/scripts/make_java_srclists.py"} ${hide;input:"build/scripts/process_command_files.py"} ${hide;input:"build/scripts/java_pack_to_file.py"} --moddir ${CURDIR} --java ${BINDIR}/all-java.srclst ${KT_SRSCLIST} ${JAVA_COVERAGE_SRCLIST} --ya-start-command-file ${ALL_JAR_SOURCES} ${ext=.java:AUTO_INPUT} ${ext=.kt:AUTO_INPUT} --ya-end-command-file
+COLLECT_JAVA_SRCLIST=${YMAKE_PYTHON} ${input:"build/scripts/make_java_srclists.py"} \
+    --moddir ${CURDIR} --java ${BINDIR}/all-java.srclst ${KT_SRSCLIST} ${JAVA_COVERAGE_SRCLIST} \
+    --ya-start-command-file ${ext=.gentar:AUTO_INPUT} ${ALL_JAR_SOURCES} ${ext=.java:AUTO_INPUT} ${ext=.kt:AUTO_INPUT} --ya-end-command-file \
+    ${hide;input:"build/scripts/process_command_files.py"} \
+    ${hide;input:"build/scripts/java_pack_to_file.py"} \
+    ${hide;input:"build/scripts/autotar_gendirs.py"}
 COLLECT_CLASSPATH=${WRITER_PY} --file ${BINDIR}/bfg.txt -m --ya-start-command-file ${rootrel:MANAGED_PEERS_CLOSURE} $KT_CLASSPATH_ITEM --ya-end-command-file
 LINK_CLASSPATH=${YMAKE_PYTHON} ${input:"build/scripts/make_manifest_from_bf.py"} ${BINDIR}/bfg.txt ${ARCADIA_BUILD_ROOT}/bfg.jar
 COMPILE_JAVA_SRCLIST=$_JAVAC_RUN_HELPER($JAVAC_CMD)
 PACK_JAR=$_PACK_JAR_HELPER($TARGET)
 LINK_JAR=${hide:JAVA_FAKEID} ${hide;kv:"p JV"} ${hide;kv:"pc light-blue"} ${hide;kv:"show_out"} ${hide;requirements:"cpu:2"} \
       && $PREPARE_JAVA_BUILD_DIRS \
-      && $EXTRACT_GENDIRS \
       && $COLLECT_JAVA_SRCLIST \
       && $JAR_GEN_SRCS \
       && $ALL_KT_COMMANDS \

build/mapping.conf.json

@@ -948,6 +948,7 @@
         "6957908969": "https://devtools-registry.s3.yandex.net/6957908969",
         "7101981380": "https://devtools-registry.s3.yandex.net/7101981380",
         "7552080809": "https://devtools-registry.s3.yandex.net/7552080809",
+        "7833244075": "https://devtools-registry.s3.yandex.net/7833244075",
         "5776379446": "https://devtools-registry.s3.yandex.net/5776379446",
         "5777100597": "https://devtools-registry.s3.yandex.net/5777100597",
         "5909067709": "https://devtools-registry.s3.yandex.net/5909067709",
@@ -960,6 +961,7 @@
         "6957906780": "https://devtools-registry.s3.yandex.net/6957906780",
         "7101978818": "https://devtools-registry.s3.yandex.net/7101978818",
         "7552077729": "https://devtools-registry.s3.yandex.net/7552077729",
+        "7833241952": "https://devtools-registry.s3.yandex.net/7833241952",
         "5776375952": "https://devtools-registry.s3.yandex.net/5776375952",
         "5777098178": "https://devtools-registry.s3.yandex.net/5777098178",
         "5909065014": "https://devtools-registry.s3.yandex.net/5909065014",
@@ -972,6 +974,7 @@
         "6957901545": "https://devtools-registry.s3.yandex.net/6957901545",
         "7101973818": "https://devtools-registry.s3.yandex.net/7101973818",
         "7552072726": "https://devtools-registry.s3.yandex.net/7552072726",
+        "7833238161": "https://devtools-registry.s3.yandex.net/7833238161",
         "5776374505": "https://devtools-registry.s3.yandex.net/5776374505",
         "5777096988": "https://devtools-registry.s3.yandex.net/5777096988",
         "5909063641": "https://devtools-registry.s3.yandex.net/5909063641",
@@ -984,6 +987,7 @@
         "6957899481": "https://devtools-registry.s3.yandex.net/6957899481",
         "7101971416": "https://devtools-registry.s3.yandex.net/7101971416",
         "7552070388": "https://devtools-registry.s3.yandex.net/7552070388",
+        "7833236145": "https://devtools-registry.s3.yandex.net/7833236145",
         "5776377955": "https://devtools-registry.s3.yandex.net/5776377955",
         "5777099502": "https://devtools-registry.s3.yandex.net/5777099502",
         "5909066324": "https://devtools-registry.s3.yandex.net/5909066324",
@@ -996,6 +1000,7 @@
         "6957903888": "https://devtools-registry.s3.yandex.net/6957903888",
         "7101976574": "https://devtools-registry.s3.yandex.net/7101976574",
         "7552075245": "https://devtools-registry.s3.yandex.net/7552075245",
+        "7833240023": "https://devtools-registry.s3.yandex.net/7833240023",
         "6391354461": "https://devtools-registry.s3.yandex.net/6391354461",
         "7805449010": "https://devtools-registry.s3.yandex.net/7805449010",
         "6990868751": "https://devtools-registry.s3.yandex.net/6990868751",
@@ -1149,14 +1154,19 @@
         "7407150115": "https://devtools-registry.s3.yandex.net/7407150115",
         "7599422144": "https://devtools-registry.s3.yandex.net/7599422144",
         "7451406900": "https://devtools-registry.s3.yandex.net/7451406900",
+        "7831958167": "https://devtools-registry.s3.yandex.net/7831958167",
         "7662667352": "https://devtools-registry.s3.yandex.net/7662667352",
         "7494722271": "https://devtools-registry.s3.yandex.net/7494722271",
+        "7831990717": "https://devtools-registry.s3.yandex.net/7831990717",
         "7662737728": "https://devtools-registry.s3.yandex.net/7662737728",
         "7494597813": "https://devtools-registry.s3.yandex.net/7494597813",
+        "7831996261": "https://devtools-registry.s3.yandex.net/7831996261",
         "7662702843": "https://devtools-registry.s3.yandex.net/7662702843",
         "7495431598": "https://devtools-registry.s3.yandex.net/7495431598",
+        "7832045657": "https://devtools-registry.s3.yandex.net/7832045657",
         "7663105272": "https://devtools-registry.s3.yandex.net/7663105272",
         "7495332394": "https://devtools-registry.s3.yandex.net/7495332394",
+        "7832062717": "https://devtools-registry.s3.yandex.net/7832062717",
         "7663137240": "https://devtools-registry.s3.yandex.net/7663137240",
         "7495957585": "https://devtools-registry.s3.yandex.net/7495957585",
         "6048579718": "https://devtools-registry.s3.yandex.net/6048579718",
@@ -2113,6 +2123,7 @@
         "6957908969": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz",
         "7101981380": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz",
         "7552080809": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz",
+        "7833244075": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz",
         "5776379446": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz",
         "5777100597": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz",
         "5909067709": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz",
@@ -2125,6 +2136,7 @@
         "6957906780": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz",
         "7101978818": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz",
         "7552077729": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz",
+        "7833241952": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz",
         "5776375952": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz",
         "5777098178": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz",
         "5909065014": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz",
@@ -2137,6 +2149,7 @@
         "6957901545": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz",
         "7101973818": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz",
         "7552072726": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz",
+        "7833238161": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz",
         "5776374505": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz",
         "5777096988": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz",
         "5909063641": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz",
@@ -2149,6 +2162,7 @@
         "6957899481": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz",
         "7101971416": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz",
         "7552070388": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz",
+        "7833236145": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz",
         "5776377955": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz",
         "5777099502": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz",
         "5909066324": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz",
@@ -2161,6 +2175,7 @@
         "6957903888": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz",
         "7101976574": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz",
         "7552075245": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz",
+        "7833240023": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz",
         "6391354461": "none-none-none-result_resources/protoc-linux-x86_64.tgz",
         "7805449010": "none-none-none-result_resources/protoc-linux-x86_64.tgz",
         "6990868751": "none-none-none-sandbox/backup/3527d100-e2d0-4b0e-bb7a-905010853d98/yfm-docs.tar",
@@ -2314,14 +2329,19 @@
         "7407150115": "ynd-clang-18-optimized-mingw-w64-x86_64-56872a411472c7ec2b09f2c8372ea4c67d069a5b",
         "7599422144": "ynd-clang-18-optimized-mingw-w64-x86_64-c40775195488adad9218aa6c6dcaf4c20dca0084",
         "7451406900": "ynd-clang-18-optimized-mingw-w64-x86_64-c66062833fd14752a7ad8fbfb8f25f370b097278",
+        "7831958167": "ynd-clang-format-18-darwin-arm64-3ac3b151b06039ed3a36becac566665d5fafefa4",
         "7662667352": "ynd-clang-format-18-darwin-arm64-809bd279d61c38bc40e133d3013ac46e8c4049b7",
         "7494722271": "ynd-clang-format-18-darwin-arm64-d0b9255e235e80974f9dfd0ebff3850929bed981",
+        "7831990717": "ynd-clang-format-18-darwin-x86_64-3ac3b151b06039ed3a36becac566665d5fafefa4",
         "7662737728": "ynd-clang-format-18-darwin-x86_64-809bd279d61c38bc40e133d3013ac46e8c4049b7",
         "7494597813": "ynd-clang-format-18-darwin-x86_64-d0b9255e235e80974f9dfd0ebff3850929bed981",
+        "7831996261": "ynd-clang-format-18-linux-aarch64-3ac3b151b06039ed3a36becac566665d5fafefa4",
         "7662702843": "ynd-clang-format-18-linux-aarch64-809bd279d61c38bc40e133d3013ac46e8c4049b7",
         "7495431598": "ynd-clang-format-18-linux-aarch64-d0b9255e235e80974f9dfd0ebff3850929bed981",
+        "7832045657": "ynd-clang-format-18-linux-x86_64-3ac3b151b06039ed3a36becac566665d5fafefa4",
         "7663105272": "ynd-clang-format-18-linux-x86_64-809bd279d61c38bc40e133d3013ac46e8c4049b7",
         "7495332394": "ynd-clang-format-18-linux-x86_64-d0b9255e235e80974f9dfd0ebff3850929bed981",
+        "7832062717": "ynd-clang-format-18-mingw-w64-x86_64-3ac3b151b06039ed3a36becac566665d5fafefa4",
         "7663137240": "ynd-clang-format-18-mingw-w64-x86_64-809bd279d61c38bc40e133d3013ac46e8c4049b7",
         "7495957585": "ynd-clang-format-18-mingw-w64-x86_64-d0b9255e235e80974f9dfd0ebff3850929bed981",
         "6048579718": "yt/go/ytrecipe/cmd/ytexec for linux",

build/platform/java/jdk/jdk11/jdk.json

@@ -1,19 +1,19 @@
 {
   "by_platform": {
     "darwin-arm64": {
-      "uri": "sbr:6936092311"
+      "uri": "sbr:7833244075"
     },
     "darwin-x86_64": {
-      "uri": "sbr:6936091858"
+      "uri": "sbr:7833241952"
     },
     "linux-x86_64": {
-      "uri": "sbr:6936090488"
+      "uri": "sbr:7833236145"
     },
     "linux-aarch64": {
-      "uri": "sbr:6936090996"
+      "uri": "sbr:7833238161"
     },
     "win32-x86_64": {
-      "uri": "sbr:6936091456"
+      "uri": "sbr:7833240023"
     }
   }
 }

build/plugins/pybuild.py

@@ -8,7 +8,6 @@
 from _common import stripext, rootrel_arc_src, listid, pathid, lazy, get_no_lint_value, ugly_conftest_exception
 
 
-YA_IDE_VENV_VAR = 'YA_IDE_VENV'
 PY_NAMESPACE_PREFIX = 'py/namespace'
 BUILTIN_PROTO = 'builtin_proto'
 DEFAULT_FLAKE8_FILE_PROCESSING_TIME = "1.5"  # in seconds
@@ -37,8 +36,8 @@ def is_extended_source_search_enabled(path, unit):
         return False
     if unit.get('NO_EXTENDED_SOURCE_SEARCH') == 'yes':
         return False
-    # contrib is unfriendly to extended source search
-    if unit.resolve_arc_path(path).startswith('$S/contrib/'):
+    # Python itself and contrib/python are unfriendly to extended source search
+    if unit.resolve_arc_path(path).startswith(('$S/contrib/python', '$S/contrib/tools/python3')):
         return False
     return True
 
@@ -252,7 +251,7 @@ def onpy_srcs(unit, *args):
     with_py = not unit.get('PYBUILD_NO_PY')
     with_pyc = not unit.get('PYBUILD_NO_PYC')
     in_proto_library = unit.get('PY_PROTO') or unit.get('PY3_PROTO')
-    venv = unit.get(YA_IDE_VENV_VAR)
+    external_py_files_mode = unit.get('EXTERNAL_PY_FILES') or unit.get('YA_IDE_VENV')
     need_gazetteer_peerdir = False
     trim = 0
 
@@ -542,10 +541,21 @@ def process_pyx(filename, path, out_suffix, with_ext):
         if py3:
             mod_list_md5 = md5()
             compress = False
+            resfs_mocks = []
+
             for path, mod in pys:
                 mod_list_md5.update(six.ensure_binary(mod))
-                if not (venv and is_extended_source_search_enabled(path, unit)):
-                    dest = 'py/' + mod.replace('.', '/') + '.py'
+                dest = 'py/' + mod.replace('.', '/') + '.py'
+                # In external_py_files mode we want to build python binaries without embedded python files.
+                # The application will still be able to load them from the file system.
+                # However, the import hook still needs meta information about the file locations to work correctly.
+                # That's why we manually register the files in resfs/src/resfs/file, but don't provide any actual files.
+                # For more info see:
+                # - library/python/runtime_py3
+                # - https://docs.yandex-team.ru/ya-make/manual/python/vars
+                if external_py_files_mode and is_extended_source_search_enabled(path, unit):
+                    resfs_mocks += ['-', 'resfs/src/resfs/file/' + dest + '=' + rootrel_arc_src(path, unit)]
+                else:
                     if with_py:
                         res += ['DEST', dest, path]
                     if with_pyc:
@@ -556,6 +566,9 @@ def process_pyx(filename, path, out_suffix, with_ext):
                     if not compress and ugly_conftest_exception(path):
                         compress = True
 
+            if resfs_mocks:
+                unit.onresource(['DONT_COMPRESS'] + resfs_mocks)
+
             if py_namespaces:
                 # Note: Add md5 to key to prevent key collision if two or more PY_SRCS() used in the same ya.make
                 ns_res = ['DONT_COMPRESS']

build/plugins/ytest.py

@@ -790,13 +790,21 @@ def onadd_check_py_imports(fields, unit, *args):
     if unit.get("TIDY") == "yes":
         # graph changed for clang_tidy tests
         return
+
     if unit.get('NO_CHECK_IMPORTS_FOR_VALUE').strip() == "":
         return
+
     unit.onpeerdir(['library/python/testing/import_test'])
 
     dart_record = create_dart_record(fields, unit, (), {})
     dart_record[df.TestName.KEY] = 'pyimports'
     dart_record[df.ScriptRelPath.KEY] = 'py.imports'
+    # Import tests work correctly in this mode, but can slow down by 2-3 times,
+    # due to the fact that files need to be read from the file system.
+    # Therefore, we disable them, since the external-py-files mode is designed exclusively
+    # to speed up the short cycle of developing regular tests.
+    if unit.get('EXTERNAL_PY_FILES'):
+        dart_record[df.SkipTest.KEY] = 'Import tests disabled in external-py-files mode'
 
     data = dump_test(unit, dart_record)
     if data: