ARROW-3759: [R][CI] Build and test (no libarrow) on Windows in Appveyor

appveyor.yml

@@ -19,7 +19,7 @@
 os: Visual Studio 2015
 
 only_commits:
-  # Skip commits not related to Python, C++, C#, Go, Ruby or Rust
+  # Skip commits not related to Python, C++, C#, Go, R, Ruby, or Rust
   files:
     - appveyor.yml
     - c_glib/
@@ -29,6 +29,7 @@ only_commits:
     - format/
     - go/
     - python/
+    - r/
     - ruby/
     - rust/
 
@@ -102,6 +103,8 @@ environment:
       GO111MODULE: on
       GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64'
       PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%'
+    - JOB: "R without libarrow"
+      USE_CLCACHE: false
 
   MSVC_DEFAULT_OPTIONS: ON
   APPVEYOR_SAVE_CACHE_ON_ERROR: true
@@ -119,6 +122,57 @@ before_build:
 build_script:
   - call ci\appveyor-build.bat
 
+for:
+-
+  matrix:
+    only:
+      - JOB: "R without libarrow"
+  init:
+    ps: |
+          $ErrorActionPreference = "Stop"
+          Invoke-WebRequest http://raw.githubusercontent.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1"
+          Import-Module '..\appveyor-tool.ps1'
+
+  before_build:
+    - cd r
+    - ps: Bootstrap
+    - cd %APPVEYOR_BUILD_FOLDER%
+
+  build_script:
+    # TODO: for the version of the R tests that builds, probably can't use this
+    # ci script because we need to build with the Rtools toolchain
+    # - call ci\appveyor-build.bat
+    - pushd r
+    - travis-tool.sh install_deps
+
+  test_script:
+    - travis-tool.sh run_tests
+
+  on_failure:
+    - travis-tool.sh dump_logs
+
+  environment:
+    NOT_CRAN: true
+
+  artifacts:
+    - path: 'r\*.Rcheck\**\*.log'
+      name: Logs
+
+    - path: 'r\*.Rcheck\**\*.out'
+      name: Logs
+
+    - path: 'r\*.Rcheck\**\*.fail'
+      name: Logs
+
+    - path: 'r\*.Rcheck\**\*.Rout'
+      name: Logs
+
+    - path: 'r\*_*.tar.gz'
+      name: Bits
+
+    - path: 'r\*_*.zip'
+      name: Bits
+
 # Disable test discovery
 test: off
 

ci/appveyor-filter-changes.bat

@@ -43,6 +43,13 @@ if "%JOB%" == "Rust" (
         echo ===
         appveyor exit
     )
+) else if "%JOB:~,2%" == "R " (
+    if "%ARROW_CI_R_AFFECTED%" == "0" (
+        echo ===
+        echo === No C++ or R changes, exiting job
+        echo ===
+        appveyor exit
+    )
 ) else (
     if "%ARROW_CI_PYTHON_AFFECTED%" == "0" (
         echo ===

dev/release/00-prepare-test.rb

@@ -130,13 +130,6 @@ def test_update_version_pre_tag
                         "+Version: #{@release_version}"],
                      ],
                    },
-                   {
-                     path: "r/src/Makevars.win",
-                     hunks: [
-                       ["-VERSION = #{@previous_version}.9000",
-                        "+VERSION = #{@release_version}"],
-                     ],
-                   },
                    {
                      path: "ruby/red-arrow-cuda/lib/arrow-cuda/version.rb",
                      hunks: [
@@ -279,13 +272,6 @@ def test_update_version_post_tag
                         "+Version: #{@release_version}.9000"],
                      ],
                    },
-                   {
-                     path: "r/src/Makevars.win",
-                     hunks: [
-                       ["-VERSION = #{@release_version}",
-                        "+VERSION = #{@release_version}.9000"],
-                     ],
-                   },
                    {
                      path: "ruby/red-arrow-cuda/lib/arrow-cuda/version.rb",
                      hunks: [

dev/release/00-prepare.sh

@@ -101,14 +101,6 @@ update_versions() {
   git add DESCRIPTION
   cd -
 
-  cd "${SOURCE_DIR}/../../r/src"
-  sed -i.bak -E -e \
-    "s/^VERSION = .+/VERSION = ${r_version}/" \
-    Makevars.win
-  rm -f Makevars.win.bak
-  git add Makevars.win
-  cd -
-
   cd "${SOURCE_DIR}/../../ruby"
   sed -i.bak -E -e \
     "s/^  VERSION = \".+\"/  VERSION = \"${version}\"/g" \

r/configure.win

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# generate code
+if [ "$ARROW_R_DEV" == "TRUE" ]; then
+  echo "*** Generating code with data-raw/codegen.R"
+  "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" data-raw/codegen.R
+fi
+
+VERSION=$(grep ^Version DESCRIPTION | sed s/Version:\ //)
+# Try to find/download a C++ Arrow binary
+"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" "tools/winlibs.R" $VERSION
+# If binary not found, script exits nonzero
+if [ $? -ne 0 ]; then
+  # We'll have to install without libarrow. Note to the user how to rectify.
+  echo "------------------------- NOTE ---------------------------"
+  echo "After installation, please run arrow::install_arrow() to install"
+  echo "required runtime libraries"
+  echo "---------------------------------------------------------"
+else
+  # Set the right flags to point to and enable arrow/parquet
+  RWINLIB="../windows/arrow-${VERSION}"
+  PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_R_WITH_PARQUET -DARROW_R_WITH_ARROW"
+  PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow -lthrift -lboost_regex-mt-s -ldouble-conversion -lz -lws2_32"
+fi
+
+echo "*** Writing Makevars.win"
+sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" src/Makevars.in > src/Makevars.win
+# Success
+exit 0

r/tools/winlibs.R

@@ -17,10 +17,10 @@
 
 # Download static arrow from rwinlib
 VERSION <- commandArgs(TRUE)
-if(!file.exists(sprintf("../windows/arrow-%s/include/arrow/api.h", VERSION))){
+if(!file.exists(sprintf("windows/arrow-%s/include/arrow/api.h", VERSION))){
   if(getRversion() < "3.3.0") setInternet2()
   download.file(sprintf("https://github.com/rwinlib/arrow/archive/v%s.zip", VERSION), "lib.zip", quiet = TRUE)
-  dir.create("../windows", showWarnings = FALSE)
-  unzip("lib.zip", exdir = "../windows")
+  dir.create("windows", showWarnings = FALSE)
+  unzip("lib.zip", exdir = "windows")
   unlink("lib.zip")
 }