HDFS-16026. Restore cross platform mkstemp

dev-support/Jenkinsfile

@@ -57,6 +57,116 @@ pipeline {
             }
         }
 
+        stage ('precommit-run (centos-7)') {
+            steps {
+                withCredentials(
+                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+                                  passwordVariable: 'GITHUB_TOKEN',
+                                  usernameVariable: 'GITHUB_USER'),
+                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+                                        passwordVariable: 'JIRA_PASSWORD',
+                                        usernameVariable: 'JIRA_USER')]) {
+                        sh '''#!/usr/bin/env bash
+
+                        set -e -x
+
+                        TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
+
+                        # this must be clean for every run
+                        if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
+                          rm -rf "${WORKSPACE}/${PATCHDIR}"
+                        fi
+                        mkdir -p "${WORKSPACE}/${PATCHDIR}"
+
+                        # if given a JIRA issue, process it. If CHANGE_URL is set
+                        # (e.g., Github Branch Source plugin), process it.
+                        # otherwise exit, because we don't want Hadoop to do a
+                        # full build.  We wouldn't normally do this check for smaller
+                        # projects. :)
+                        if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
+                            YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
+                        elif [[ -z "${CHANGE_URL}" ]]; then
+                            echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
+                            exit 0
+                        fi
+
+                        YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
+
+                        # where the source is located
+                        YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
+
+                        # our project defaults come from a personality file
+                        YETUS_ARGS+=("--project=hadoop")
+                        YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
+
+                        # lots of different output formats
+                        YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
+                        YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
+                        YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
+
+                        # enable writing back to Github
+                        YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
+
+                        # enable writing back to ASF JIRA
+                        YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
+                        YETUS_ARGS+=(--jira-user="${JIRA_USER}")
+
+                        # auto-kill any surefire stragglers during unit test runs
+                        YETUS_ARGS+=("--reapermode=kill")
+
+                        # set relatively high limits for ASF machines
+                        # changing these to higher values may cause problems
+                        # with other jobs on systemd-enabled machines
+                        YETUS_ARGS+=("--proclimit=5500")
+                        YETUS_ARGS+=("--dockermemlimit=22g")
+
+                        # -1 spotbugs issues that show up prior to the patch being applied
+                        YETUS_ARGS+=("--spotbugs-strict-precheck")
+
+                        # rsync these files back into the archive dir
+                        YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
+
+                        # URL for user-side presentation in reports and such to our artifacts
+                        # (needs to match the archive bits below)
+                        YETUS_ARGS+=("--build-url-artifacts=artifact/out")
+
+                        # plugins to enable
+                        YETUS_ARGS+=("--plugins=all")
+
+                        # don't let these tests cause -1s because we aren't really paying that
+                        # much attention to them
+                        YETUS_ARGS+=("--tests-filter=checkstyle")
+
+                        # run in docker mode and specifically point to our
+                        # Dockerfile since we don't want to use the auto-pulled version.
+                        YETUS_ARGS+=("--docker")
+                        YETUS_ARGS+=("--dockerfile=${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7")
+                        YETUS_ARGS+=("--mvn-custom-repos")
+
+                        # effectively treat dev-suport as a custom maven module
+                        YETUS_ARGS+=("--skip-dirs=dev-support")
+
+                        # help keep the ASF boxes clean
+                        YETUS_ARGS+=("--sentinel")
+
+                        # test with Java 8 and 11
+                        YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
+                        YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
+                        YETUS_ARGS+=("--multijdktests=compile")
+
+                        # custom javadoc goals
+                        YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork")
+
+                        # write Yetus report as GitHub comment (YETUS-1102)
+                        YETUS_ARGS+=("--github-write-comment")
+                        YETUS_ARGS+=("--github-use-emoji-vote")
+
+                        bash -x "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
+                        '''
+                }
+            }
+        }
+
         stage ('precommit-run') {
             steps {
                 withCredentials(

dev-support/docker/Dockerfile_centos_7

@@ -24,42 +24,67 @@ WORKDIR /root
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN yum update -y \
-    && yum install -y centos-release-scl \
-    && yum install -y devtoolset-9 \
     && yum install -y \
-    ant \
-    build-essential \
-    bzip2 \
-    bzip2-devel \
-    clang \
-    curl \
-    cyrus-sasl-devel \
-    doxygen \
-    fuse \
-    fuse-libs \
-    fuse-devel \
-    git \
-    libcurl-devel \
-    libtirpc-devel \
-    libpmem-devel \
-    libtool \
-    lz4-devel \
-    make \
-    openssl-devel \
-    pinentry-curses \
-    python3 \
-    python3-pip \
-    python3-setuptools \
-    python3-wheel \
-    rsync \
-    snappy-devel \
-    sudo \
-    valgrind \
-    zlib-devel
-
-# Set GCC 9 as the default C/C++ compiler
-RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
-SHELL ["/bin/bash", "--login", "-c"]
+        ant \
+        build-essential \
+        bzip2 \
+        bzip2-devel \
+        clang \
+        curl \
+        cyrus-sasl-devel \
+        dnf \
+        doxygen \
+        fuse \
+        fuse-libs \
+        fuse-devel \
+        git \
+        libcurl-devel \
+        libtirpc-devel \
+        libpmem-devel \
+        libtool \
+        lz4-devel \
+        make \
+        openssl-devel \
+        pinentry-curses \
+        python3 \
+        python3-pip \
+        python3-setuptools \
+        python3-wheel \
+        rsync \
+        snappy-devel \
+        sudo \
+        valgrind \
+        zlib-devel
+
+RUN dnf install -y \
+    gcc \
+    gcc-c++ \
+    gmp-devel \
+    mpfr-devel \
+    libmpc-devel
+
+####
+# Install GCC 9.3.0
+####
+# hadolint ignore=DL3003
+RUN mkdir -p /tmp/gcc /opt/gcc "${HOME}/.gcc" \
+    && curl -L -s -S https://ftp.gnu.org/gnu/gcc/gcc-9.3.0/gcc-9.3.0.tar.gz  -o /tmp/gcc/gcc-9.3.0.tar.gz \
+    && tar xzf /tmp/gcc/gcc-9.3.0.tar.gz --strip-components 1 -C /opt/gcc \
+    && cd /opt/gcc || exit \
+    && ./configure --enable-languages=c,c++ --disable-multilib --prefix=${HOME}/.gcc/9.3.0 \
+    && make -j1 \
+    && make install \
+    && cd /root || exit
+RUN echo "export CC=${HOME}/.gcc/9.3.0/bin/gcc" >> /etc/bashrc
+RUN echo "export CXX=${HOME}/.gcc/9.3.0/bin/g++" >> /etc/bashrc
+RUN echo "export PATH=${HOME}/.gcc/9.3.0/bin:${PATH}" >> /etc/bashrc
+RUN echo "export LD_LIBRARY_PATH=${HOME}/.gcc/9.3.0/lib64:${LD_LIRBARY_PATH}" >> /etc/bashrc
+ENV CC "${HOME}/.gcc/9.3.0/bin/gcc"
+ENV CXX "${HOME}/.gcc/9.3.0/bin/g++"
+ENV PATH "${HOME}/.gcc/9.3.0/bin:${PATH}"
+ENV LD_LIBRARY_PATH "${HOME}/.gcc/9.3.0/lib64:${LD_LIRBARY_PATH}"
+
+SHELL ["/bin/bash", "-l", "-c"]
 
 ####
 # Install Maven 3.6.3

hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt

@@ -91,7 +91,8 @@ function(build_libhdfs_test NAME LIBRARY)
             list(APPEND FILES ${CMAKE_SOURCE_DIR}/main/native/libhdfs-tests/${FIL})
         endif()
     endforeach()
-    add_executable("${NAME}_${LIBRARY}" ${FILES})
+    add_executable("${NAME}_${LIBRARY}" $<TARGET_OBJECTS:x_platform_obj_c_api> $<TARGET_OBJECTS:x_platform_obj> ${FILES})
+    target_include_directories("${NAME}_${LIBRARY}" PRIVATE main/native/libhdfspp/lib)
 endfunction()
 
 function(add_libhdfs_test NAME LIBRARY)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c

@@ -22,6 +22,8 @@
 #include "hdfspp/hdfs_ext.h"
 #include "native_mini_dfs.h"
 #include "os/thread.h"
+#include "x-platform/c-api/syscall.h"
+
 
 #include <errno.h>
 #include <inttypes.h>
@@ -126,7 +128,8 @@ static int hdfsCurlData(const char *host, const tPort port, const char *dirNm,
   EXPECT_NONNULL(pw = getpwuid(uid));
 
   int fd = -1;
-  EXPECT_NONNEGATIVE(fd = mkstemp(tmpFile));
+  EXPECT_NONNEGATIVE(fd = x_platform_syscall_create_and_open_temp_file(
+                         tmpFile, sizeof tmpFile));
 
   tSize sz = 0;
   while (sz < fileSz) {

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.cc

@@ -18,6 +18,26 @@
 
 #include "x-platform/syscall.h"
 
-extern "C" int x_platform_syscall_write_to_stdout(const char* msg) {
+#include <algorithm>
+#include <vector>
+
+extern "C" {
+int x_platform_syscall_write_to_stdout(const char* msg) {
   return XPlatform::Syscall::WriteToStdout(msg) ? 1 : 0;
 }
+
+int x_platform_syscall_create_and_open_temp_file(char* pattern,
+                                                 const size_t pattern_len) {
+  std::vector<char> pattern_vec(pattern, pattern + pattern_len);
+
+  const auto fd = XPlatform::Syscall::CreateAndOpenTempFile(pattern_vec);
+  if (fd != -1) {
+    std::copy_n(pattern_vec.begin(), pattern_len, pattern);
+  }
+  return fd;
+}
+
+int x_platform_syscall_close_file(const int fd) {
+  return XPlatform::Syscall::CloseFile(fd);
+}
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h

@@ -24,5 +24,8 @@
  */
 
 int x_platform_syscall_write_to_stdout(const char* msg);
+int x_platform_syscall_create_and_open_temp_file(char* pattern,
+                                                 size_t pattern_len);
+int x_platform_syscall_close_file(int fd);
 
 #endif  // NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_SYSCALL_H

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h

@@ -20,6 +20,7 @@
 #define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_SYSCALL
 
 #include <string>
+#include <vector>
 
 /**
  * The {@link XPlatform} namespace contains components that
@@ -84,6 +85,34 @@ class Syscall {
   static bool StringCompareIgnoreCase(const std::string& a,
                                       const std::string& b);
 
+  /**
+   * Creates and opens a temporary file with a given {@link pattern}.
+   * The {@link pattern} must end with a minimum of 6 'X' characters.
+   * This function will first modify the last 6 'X' characters with
+   * random character values, which serve as the temporary file name.
+   * Subsequently opens the file and returns the file descriptor for
+   * the same. The behaviour of this function is the same as that of
+   * POSIX mkstemp function. The file must be later closed by the
+   * application and is not handled by this function.
+   *
+   * @param pattern the pattern to be used for the temporary filename.
+   * @returns an integer representing the file descriptor for the
+   * opened temporary file. Returns -1 in the case of error and sets
+   * the global errno with the appropriate error code.
+   */
+  static int CreateAndOpenTempFile(std::vector<char>& pattern);
+
+  /**
+   * Closes the file corresponding to given {@link file_descriptor}.
+   *
+   * @param file_descriptor the file descriptor of the file to close.
+   * @returns a boolean indicating the status of the call to this
+   * function. true if it's a success, false in the case of an error.
+   * The global errno is set if the call to this function was not
+   * successful.
+   */
+  static bool CloseFile(int file_descriptor);
+
  private:
   static bool WriteToStdoutImpl(const char* message);
 };

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc

@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include <cstring>
+#include <vector>
 
 #include "syscall.h"
 
@@ -59,3 +60,13 @@ bool XPlatform::Syscall::StringCompareIgnoreCase(const std::string& a,
                                                  const std::string& b) {
   return strcasecmp(a.c_str(), b.c_str()) == 0;
 }
+
+int XPlatform::Syscall::CreateAndOpenTempFile(std::vector<char>& pattern) {
+  // Make space for mkstemp to add NULL character at the end
+  pattern.resize(pattern.size() + 1);
+  return mkstemp(pattern.data());
+}
+
+bool XPlatform::Syscall::CloseFile(const int file_descriptor) {
+  return close(file_descriptor) == 0;
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc

@@ -19,7 +19,14 @@
 #include <Shlwapi.h>
 #include <WinBase.h>
 #include <Windows.h>
+#include <fcntl.h>
+#include <io.h>
+#include <share.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 
+#include <cerrno>
+#include <cstdlib>
 #include <cstring>
 
 #include "syscall.h"
@@ -64,3 +71,26 @@ bool XPlatform::Syscall::StringCompareIgnoreCase(const std::string& a,
                                                  const std::string& b) {
   return _stricmp(a.c_str(), b.c_str()) == 0;
 }
+
+int XPlatform::Syscall::CreateAndOpenTempFile(std::vector<char>& pattern) {
+  if (_set_errno(0) != 0) {
+    return -1;
+  }
+
+  // Make space for _mktemp_s to add NULL character at the end
+  pattern.resize(pattern.size() + 1);
+  if (_mktemp_s(pattern.data(), pattern.size()) != 0) {
+    return -1;
+  }
+
+  auto fd{-1};
+  if (_sopen_s(&fd, pattern.data(), _O_RDWR | _O_CREAT | _O_EXCL, _SH_DENYNO,
+               _S_IREAD | _S_IWRITE) != 0) {
+    return -1;
+  }
+  return fd;
+}
+
+bool XPlatform::Syscall::CloseFile(const int file_descriptor) {
+  return _close(file_descriptor) == 0;
+}