[Fix] Correctly set testcase build url and gn args for fuzzing on batch

src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py

@@ -1002,17 +1002,18 @@ def create_testcase(group: uworker_msg_pb2.FuzzTaskCrashGroup,
                     fully_qualified_fuzzer_name: str):
   """Create a testcase based on crash."""
   crash = group.main_crash
+  fuzz_task_output = uworker_output.fuzz_task_output
   comment = (f'Fuzzer {fully_qualified_fuzzer_name} generated testcase crashed '
              f'in {crash.crash_time} seconds '
-             f'(r{uworker_output.fuzz_task_output.crash_revision})')
+             f'(r{fuzz_task_output.crash_revision})')
   testcase_id = data_handler.store_testcase(
       crash=crash,
       fuzzed_keys=crash.fuzzed_key or None,
       minimized_keys=get_fixed_or_minimized_key(group.one_time_crasher_flag),
       regression=get_regression(group.one_time_crasher_flag),
       fixed=get_fixed_or_minimized_key(group.one_time_crasher_flag),
       one_time_crasher_flag=group.one_time_crasher_flag,
-      crash_revision=int(uworker_output.fuzz_task_output.crash_revision),
+      crash_revision=int(fuzz_task_output.crash_revision),
       comment=comment,
       absolute_path=crash.absolute_path,
       fuzzer_name=uworker_input.fuzzer_name,
@@ -1036,6 +1037,15 @@ def create_testcase(group: uworker_msg_pb2.FuzzTaskCrashGroup,
       events.TestcaseCreationEvent(
           testcase=testcase, creation_origin=events.TestcaseOrigin.FUZZ_TASK))
 
+  # Add build metadata to testcase. This is needed due to some build env vars
+  # not being propagated if utask is not executed locally (e.g., BUILD_URL).
+  data_handler.set_build_metadata_to_testcase(
+      testcase,
+      build_key=fuzz_task_output.build_key,
+      build_url=fuzz_task_output.build_url,
+      gn_args=fuzz_task_output.gn_args,
+      update=True)
+
   if group.context.fuzzer_metadata:
     for key, value in group.context.fuzzer_metadata.items():
       testcase.set_metadata(key, value, update_testcase=False)
@@ -1879,6 +1889,7 @@ def run(self):
     fuzz_target = self.fuzz_target.binary if self.fuzz_target else None
     build_setup_result = build_manager.setup_build(
         environment.get_value('APP_REVISION'), fuzz_target=fuzz_target)
+    _add_build_metadata_to_output(self.fuzz_task_output)
 
     engine_impl = engine.get(self.fuzzer.name)
     if engine_impl and build_setup_result:
@@ -2241,6 +2252,14 @@ def _to_engine_output(output: str, crash_path: str, return_code: int,
   return engine_output
 
 
+def _add_build_metadata_to_output(
+    fuzz_task_output: uworker_msg_pb2.FuzzTaskOutput) -> None:
+  """Add build metadata from environment to fuzz task output."""
+  fuzz_task_output.build_key = environment.get_value('BUILD_KEY', '')
+  fuzz_task_output.build_url = environment.get_value('BUILD_URL', '')
+  fuzz_task_output.gn_args = data_handler.get_filtered_gn_args() or ''
+
+
 def _upload_engine_output(engine_output):
   timestamp = uworker_io.proto_timestamp_to_timestamp(engine_output.timestamp)
   testcase_manager.upload_log(engine_output.output.decode(),

src/clusterfuzz/_internal/datastore/data_handler.py

@@ -834,16 +834,8 @@ def store_testcase(crash, fuzzed_keys, minimized_keys, regression, fixed,
   return testcase_id
 
 
-def set_initial_testcase_metadata(testcase):
-  """Set various testcase metadata fields during testcase initialization."""
-  build_key = environment.get_value('BUILD_KEY')
-  if build_key:
-    testcase.set_metadata('build_key', build_key, update_testcase=False)
-
-  build_url = environment.get_value('BUILD_URL')
-  if build_url:
-    testcase.set_metadata('build_url', build_url, update_testcase=False)
-
+def get_filtered_gn_args() -> str | None:
+  """Return filtered GN args based on filepath GN_ARGS_PATH."""
   gn_args_path = environment.get_value('GN_ARGS_PATH', '')
   if gn_args_path:
     gn_args = utils.read_data_from_file(
@@ -856,8 +848,41 @@ def set_initial_testcase_metadata(testcase):
         if not GOMA_DIR_LINE_REGEX.match(line)
     ]
     filtered_gn_args = '\n'.join(filtered_gn_args_lines)
-    testcase.set_metadata('gn_args', filtered_gn_args, update_testcase=False)
+    return filtered_gn_args
+  return None
+
+
+def set_build_metadata_to_testcase(testcase: data_types.Testcase,
+                                   build_key: str | None = None,
+                                   build_url: str | None = None,
+                                   gn_args: str | None = None,
+                                   update: bool = False):
+  """Set testcase metadata fields related to the build metadata."""
+  dirty_flag = False
+  build_key = (
+      environment.get_value('BUILD_KEY') if build_key is None else build_key)
+  if build_key and not testcase.get_metadata('build_key'):
+    dirty_flag = True
+    testcase.set_metadata('build_key', build_key, update_testcase=False)
+
+  build_url = (
+      environment.get_value('BUILD_URL') if build_url is None else build_url)
+  if build_url and not testcase.get_metadata('build_url'):
+    dirty_flag = True
+    testcase.set_metadata('build_url', build_url, update_testcase=False)
+
+  gn_args = get_filtered_gn_args() if gn_args is None else gn_args
+  if gn_args and not testcase.get_metadata('gn_args'):
+    dirty_flag = True
+    testcase.set_metadata('gn_args', gn_args, update_testcase=False)
 
+  if update and dirty_flag:
+    testcase.put()
+
+
+def set_initial_testcase_metadata(testcase: data_types.Testcase) -> None:
+  """Set various testcase metadata fields during testcase initialization."""
+  set_build_metadata_to_testcase(testcase)
   testcase.platform = environment.platform().lower()
   testcase.platform_id = environment.get_platform_id()
 

src/clusterfuzz/_internal/protos/uworker_msg.proto

@@ -261,6 +261,9 @@ message FuzzTaskOutput {
   optional BuildData build_data = 14;
   optional int64 app_revision = 15;
   repeated EngineOutput engine_outputs = 16;
+  optional string build_key = 17;
+  optional string build_url = 18;
+  optional string gn_args = 19;
 }
 
 message MinimizeTaskOutput {

src/clusterfuzz/_internal/protos/uworker_msg_pb2.pyi

@@ -1250,6 +1250,9 @@ class FuzzTaskOutput(google.protobuf.message.Message):
     BUILD_DATA_FIELD_NUMBER: builtins.int
     APP_REVISION_FIELD_NUMBER: builtins.int
     ENGINE_OUTPUTS_FIELD_NUMBER: builtins.int
+    BUILD_KEY_FIELD_NUMBER: builtins.int
+    BUILD_URL_FIELD_NUMBER: builtins.int
+    GN_ARGS_FIELD_NUMBER: builtins.int
     fully_qualified_fuzzer_name: builtins.str
     """TODO(metzman): Remove this since tworkers should know what this is based on
     the input.
@@ -1272,6 +1275,9 @@ class FuzzTaskOutput(google.protobuf.message.Message):
     app_revision: builtins.int
     @property
     def engine_outputs(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___EngineOutput]: ...
+    build_key: builtins.str
+    build_url: builtins.str
+    gn_args: builtins.str
     def __init__(
         self,
         *,
@@ -1288,14 +1294,21 @@ class FuzzTaskOutput(google.protobuf.message.Message):
         build_data: global___BuildData | None = ...,
         app_revision: builtins.int | None = ...,
         engine_outputs: collections.abc.Iterable[global___EngineOutput] | None = ...,
+        build_key: builtins.str | None = ...,
+        build_url: builtins.str | None = ...,
+        gn_args: builtins.str | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing_extensions.Literal["_app_revision", b"_app_revision", "_build_data", b"_build_data", "_crash_revision", b"_crash_revision", "_fully_qualified_fuzzer_name", b"_fully_qualified_fuzzer_name", "_fuzzer_revision", b"_fuzzer_revision", "_fuzzer_run_results", b"_fuzzer_run_results", "_job_run_timestamp", b"_job_run_timestamp", "_new_targets_count", b"_new_targets_count", "_testcases_executed", b"_testcases_executed", "app_revision", b"app_revision", "build_data", b"build_data", "crash_revision", b"crash_revision", "fully_qualified_fuzzer_name", b"fully_qualified_fuzzer_name", "fuzzer_revision", b"fuzzer_revision", "fuzzer_run_results", b"fuzzer_run_results", "job_run_timestamp", b"job_run_timestamp", "new_targets_count", b"new_targets_count", "testcases_executed", b"testcases_executed"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing_extensions.Literal["_app_revision", b"_app_revision", "_build_data", b"_build_data", "_crash_revision", b"_crash_revision", "_fully_qualified_fuzzer_name", b"_fully_qualified_fuzzer_name", "_fuzzer_revision", b"_fuzzer_revision", "_fuzzer_run_results", b"_fuzzer_run_results", "_job_run_timestamp", b"_job_run_timestamp", "_new_targets_count", b"_new_targets_count", "_testcases_executed", b"_testcases_executed", "app_revision", b"app_revision", "build_data", b"build_data", "crash_groups", b"crash_groups", "crash_revision", b"crash_revision", "engine_outputs", b"engine_outputs", "fully_qualified_fuzzer_name", b"fully_qualified_fuzzer_name", "fuzz_targets", b"fuzz_targets", "fuzzer_revision", b"fuzzer_revision", "fuzzer_run_results", b"fuzzer_run_results", "job_run_timestamp", b"job_run_timestamp", "new_targets_count", b"new_targets_count", "testcase_run_jsons", b"testcase_run_jsons", "testcases_executed", b"testcases_executed"]) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["_app_revision", b"_app_revision", "_build_data", b"_build_data", "_build_key", b"_build_key", "_build_url", b"_build_url", "_crash_revision", b"_crash_revision", "_fully_qualified_fuzzer_name", b"_fully_qualified_fuzzer_name", "_fuzzer_revision", b"_fuzzer_revision", "_fuzzer_run_results", b"_fuzzer_run_results", "_gn_args", b"_gn_args", "_job_run_timestamp", b"_job_run_timestamp", "_new_targets_count", b"_new_targets_count", "_testcases_executed", b"_testcases_executed", "app_revision", b"app_revision", "build_data", b"build_data", "build_key", b"build_key", "build_url", b"build_url", "crash_revision", b"crash_revision", "fully_qualified_fuzzer_name", b"fully_qualified_fuzzer_name", "fuzzer_revision", b"fuzzer_revision", "fuzzer_run_results", b"fuzzer_run_results", "gn_args", b"gn_args", "job_run_timestamp", b"job_run_timestamp", "new_targets_count", b"new_targets_count", "testcases_executed", b"testcases_executed"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["_app_revision", b"_app_revision", "_build_data", b"_build_data", "_build_key", b"_build_key", "_build_url", b"_build_url", "_crash_revision", b"_crash_revision", "_fully_qualified_fuzzer_name", b"_fully_qualified_fuzzer_name", "_fuzzer_revision", b"_fuzzer_revision", "_fuzzer_run_results", b"_fuzzer_run_results", "_gn_args", b"_gn_args", "_job_run_timestamp", b"_job_run_timestamp", "_new_targets_count", b"_new_targets_count", "_testcases_executed", b"_testcases_executed", "app_revision", b"app_revision", "build_data", b"build_data", "build_key", b"build_key", "build_url", b"build_url", "crash_groups", b"crash_groups", "crash_revision", b"crash_revision", "engine_outputs", b"engine_outputs", "fully_qualified_fuzzer_name", b"fully_qualified_fuzzer_name", "fuzz_targets", b"fuzz_targets", "fuzzer_revision", b"fuzzer_revision", "fuzzer_run_results", b"fuzzer_run_results", "gn_args", b"gn_args", "job_run_timestamp", b"job_run_timestamp", "new_targets_count", b"new_targets_count", "testcase_run_jsons", b"testcase_run_jsons", "testcases_executed", b"testcases_executed"]) -> None: ...
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_app_revision", b"_app_revision"]) -> typing_extensions.Literal["app_revision"] | None: ...
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_build_data", b"_build_data"]) -> typing_extensions.Literal["build_data"] | None: ...
     @typing.overload
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_build_key", b"_build_key"]) -> typing_extensions.Literal["build_key"] | None: ...
+    @typing.overload
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_build_url", b"_build_url"]) -> typing_extensions.Literal["build_url"] | None: ...
+    @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_crash_revision", b"_crash_revision"]) -> typing_extensions.Literal["crash_revision"] | None: ...
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_fully_qualified_fuzzer_name", b"_fully_qualified_fuzzer_name"]) -> typing_extensions.Literal["fully_qualified_fuzzer_name"] | None: ...
@@ -1304,6 +1317,8 @@ class FuzzTaskOutput(google.protobuf.message.Message):
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_fuzzer_run_results", b"_fuzzer_run_results"]) -> typing_extensions.Literal["fuzzer_run_results"] | None: ...
     @typing.overload
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_gn_args", b"_gn_args"]) -> typing_extensions.Literal["gn_args"] | None: ...
+    @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_job_run_timestamp", b"_job_run_timestamp"]) -> typing_extensions.Literal["job_run_timestamp"] | None: ...
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_new_targets_count", b"_new_targets_count"]) -> typing_extensions.Literal["new_targets_count"] | None: ...

src/clusterfuzz/_internal/tests/core/datastore/data_handler_test.py

@@ -40,12 +40,11 @@ def setUp(self):
     test_utils.set_up_pyfakefs(self)
     helpers.patch_environ(self)
 
-  def test_set(self):
-    """Test set everything."""
+  def test_testcase_metadata_from_env(self):
+    """Tests the initial testcase metadata set from the env vars."""
     os.environ['FAIL_RETRIES'] = '3'
     os.environ['FAIL_WAIT'] = '3'
     os.environ['BUILD_KEY'] = 'build_key_value'
-    os.environ['BUILD_KEY'] = 'build_key_value'
     os.environ['BUILD_URL'] = 'build_url_value'
     os.environ['APP_DIR'] = 'app_dir_value'
     os.environ['GN_ARGS_PATH'] = 'app_dir_value/args.gn'
@@ -66,6 +65,31 @@ def test_set(self):
         'is_asan = true\nuse_goma = true\nv8_enable_verify_heap = true',
         metadata['gn_args'])
 
+  def test_testcase_metadata_from_args(self):
+    """Tests that testcase build metadata is set from args."""
+    build_key = 'build_key_test'
+    build_url = 'build_url_test'
+    os.environ['GN_ARGS_PATH'] = 'app_dir_value/args.gn'
+    self.fs.create_file(
+        'app_dir_value/args.gn',
+        contents=('is_asan = true\n'
+                  'goma_dir = /home/user/goma\n'
+                  'use_goma = false\n'
+                  'v8_enable_verify_heap = true'))
+    gn_args = data_handler.get_filtered_gn_args()
+    del os.environ['GN_ARGS_PATH']
+
+    testcase = data_types.Testcase()
+    data_handler.set_build_metadata_to_testcase(
+        testcase, build_key=build_key, build_url=build_url, gn_args=gn_args)
+    metadata = json.loads(testcase.additional_metadata)
+
+    self.assertEqual('build_key_test', metadata['build_key'])
+    self.assertEqual('build_url_test', metadata['build_url'])
+    self.assertEqual(
+        'is_asan = true\nuse_goma = false\nv8_enable_verify_heap = true',
+        metadata['gn_args'])
+
 
 @test_utils.with_cloud_emulators('datastore')
 class DataHandlerTest(unittest.TestCase):