Enhance Java build error reporting; correctly remove multiple occurre…

…nces of a PATH entry; build codegen tools separately in yb_release.py; add missing dependency

Summary:
- Enhance Java build error reporting.
- Correctly remove multiple occurrences of a PATH entry. We remove PATH entries when we deactivate a virtualenv in order to activate a new one.
- Build codegen tools (protoc-gen-insertions, bfql_codegen) separately in yb_release.py, until we figure out the issue with the dependency graph.
- bfql_codegen should depend on pgsql_protocol_proto.
- Show Make targets in the end of yb_build.sh.

Test Plan: Jenkins

Reviewers: bharat

Reviewed By: bharat

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D4826

build-support/common-build-env.sh

@@ -768,7 +768,15 @@ build_yb_java_code_filter_save_output() {
     return $mvn_exit_code
   fi
   set -e +x
-  log "ERROR: Java build finished but build command failed so log cannot be processed"
+  log "Java build or one of its output filters failed"
+  if [[ -f $java_build_output_path ]]; then
+    log "Java build output (from '$java_build_output_path'):"
+    cat "$java_build_output_path"
+    log "(End of Java build output)"
+    rm -f "$java_build_output_path"
+  else
+    log "Java build output path file not found at '$java_build_output_path'"
+  fi
   return 1
 }
 
@@ -882,10 +890,15 @@ ssh: Could not resolve hostname build-workers-.*: Name or service not known"
 remove_path_entry() {
   expect_num_args 1 "$@"
   local path_entry=$1
-  PATH=:$PATH:
-  PATH=${PATH//:$path_entry:/:}
-  PATH=${PATH#:}
-  PATH=${PATH%:}
+  local prev_path=""
+  # Remove all occurrences of the given entry.
+  while  [[ $PATH != $prev_path ]]; do
+    prev_path=$PATH
+    PATH=:$PATH:
+    PATH=${PATH//:$path_entry:/:}
+    PATH=${PATH#:}
+    PATH=${PATH%:}
+  done
   export PATH
 }
 

build-support/yb_release.py

@@ -158,10 +158,25 @@ def main():
         if args.skip_build:
             build_cmd_list += ["--skip-build"]
         if args.build_args:
+            # TODO: run with shell=True and append build_args as is.
             build_cmd_list += args.build_args.strip().split()
 
     build_cmd_line = " ".join(build_cmd_list).strip()
     logging.info("Build command line: {}".format(build_cmd_line))
+
+    if not args.java_only and not args.skip_build:
+        # TODO: figure out the dependency issues in our CMake build instead.
+        # TODO: move this into yb_build.sh itself.
+        for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']:
+            preliminary_step_cmd_list = [
+                    arg for arg in build_cmd_list if arg != 'packaged_targets'
+                ] + ['--target', preliminary_target, '--skip-java']
+            logging.info(
+                    "Running a preliminary step to build target %s: %s",
+                    preliminary_target,
+                    " ".join(preliminary_step_cmd_list))
+            subprocess.check_call(preliminary_step_cmd_list)
+
     subprocess.check_call(build_cmd_list)
 
     if not os.path.exists(build_desc_path):

requirements.txt

@@ -1,2 +1,3 @@
 PyYAML==3.12
 six==1.11.0
+yugabyte_pycommon==1.5.0

src/yb/util/CMakeLists.txt

@@ -30,6 +30,13 @@
 # under the License.
 #
 
+#######################################
+# protoc-gen-insertions
+#######################################
+
+add_executable(protoc-gen-insertions protoc-gen-insertions.cc)
+target_link_libraries(protoc-gen-insertions gutil protobuf protoc ${YB_BASE_LIBS})
+
 #######################################
 # histogram_proto
 #######################################
@@ -303,13 +310,6 @@ if(NOT APPLE)
     rt)
 endif()
 
-#######################################
-# protoc-gen-insertions
-#######################################
-
-add_executable(protoc-gen-insertions protoc-gen-insertions.cc)
-target_link_libraries(protoc-gen-insertions gutil protobuf protoc ${YB_BASE_LIBS})
-
 #######################################
 # Unit tests
 #######################################

src/yb/util/bfql/CMakeLists.txt

@@ -16,7 +16,7 @@ add_executable(bfql_codegen
                directory.cc
                codegen.cc)
 target_link_libraries(bfql_codegen yb_util yb_common_proto consensus_metadata_proto)
-
+add_dependencies(bfql_codegen pgsql_protocol_proto)
 # Use ${CMAKE_BINARY_DIR}/bin/bfql_codegen to generate builtin function files.
 SET(BF_GENERATED_FILES
     ${CMAKE_CURRENT_BINARY_DIR}/gen_opcode_table.cc

src/yb/util/jsonb.h

@@ -16,7 +16,6 @@
 
 #include <rapidjson/document.h>
 
-#include "yb/common/ql_protocol.pb.h"
 #include "yb/common/ql_value.h"
 #include "yb/util/status.h"
 

yb_build.sh

@@ -233,6 +233,12 @@ print_report() {
       if ! is_mac; then
         print_report_line "%s" "Linuxbrew dir" "${YB_LINUXBREW_DIR:-undefined}"
       fi
+      set +u
+      local make_targets_str="${make_targets[*]}"
+      set -u
+      if [[ -n $make_targets_str ]]; then
+        print_report_line "%s" "Targets" "$make_targets_str"
+      fi
       report_time "CMake" "cmake"
       report_time "C++ compilation" "make"
       report_time "Java compilation" "java_build"