Integration of arduino-preprocessor

arduino-builder/main.go

@@ -54,6 +54,7 @@ const VERSION = "1.3.25"
 const FLAG_ACTION_COMPILE = "compile"
 const FLAG_ACTION_PREPROCESS = "preprocess"
 const FLAG_ACTION_DUMP_PREFS = "dump-prefs"
+const FLAG_ACTION_CODE_COMPLETE_AT = "code-complete-at"
 const FLAG_BUILD_OPTIONS_FILE = "build-options-file"
 const FLAG_HARDWARE = "hardware"
 const FLAG_TOOLS = "tools"
@@ -118,6 +119,7 @@ func (h *propertiesFlag) Set(value string) error {
 var compileFlag *bool
 var preprocessFlag *bool
 var dumpPrefsFlag *bool
+var codeCompleteAtFlag *string
 var buildOptionsFileFlag *string
 var hardwareFoldersFlag foldersFlag
 var toolsFoldersFlag foldersFlag
@@ -141,6 +143,7 @@ func init() {
 	compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch")
 	preprocessFlag = flag.Bool(FLAG_ACTION_PREPROCESS, false, "preprocess the given sketch")
 	dumpPrefsFlag = flag.Bool(FLAG_ACTION_DUMP_PREFS, false, "dumps build properties used when compiling")
+	codeCompleteAtFlag = flag.String(FLAG_ACTION_CODE_COMPLETE_AT, "", "output code completions for sketch at a specific location. Location format is \"file:line:col\"")
 	buildOptionsFileFlag = flag.String(FLAG_BUILD_OPTIONS_FILE, "", "Instead of specifying --"+FLAG_HARDWARE+", --"+FLAG_TOOLS+" etc every time, you can load all such options from a file")
 	flag.Var(&hardwareFoldersFlag, FLAG_HARDWARE, "Specify a 'hardware' folder. Can be added multiple times for specifying multiple 'hardware' folders")
 	flag.Var(&toolsFoldersFlag, FLAG_TOOLS, "Specify a 'tools' folder. Can be added multiple times for specifying multiple 'tools' folders")
@@ -330,7 +333,8 @@ func main() {
 
 	if *dumpPrefsFlag {
 		err = builder.RunParseHardwareAndDumpBuildProperties(ctx)
-	} else if *preprocessFlag {
+	} else if *preprocessFlag || *codeCompleteAtFlag != "" {
+		ctx.CodeCompleteAt = *codeCompleteAtFlag
 		err = builder.RunPreprocess(ctx)
 	} else {
 		if flag.NArg() == 0 {

builder.go

@@ -89,7 +89,7 @@ func (s *Builder) Run(ctx *types.Context) error {
 		&WarnAboutArchIncompatibleLibraries{},
 
 		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, "Generating function prototypes..."),
-		&ContainerAddPrototypes{},
+		&PreprocessSketch{},
 
 		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, "Compiling sketch..."),
 		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_SKETCH_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
@@ -158,7 +158,7 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 
 		&WarnAboutArchIncompatibleLibraries{},
 
-		&ContainerAddPrototypes{},
+		&PreprocessSketch{},
 
 		&PrintPreprocessedSource{},
 	}

hardware/platform.txt

@@ -1,3 +1,10 @@
+# arduino-preprocessor
+# --------------------
+
+tools.arduino-preprocessor.path={runtime.tools.arduino-preprocessor.path}
+tools.arduino-preprocessor.cmd.path={path}/arduino-preprocessor
+tools.arduino-preprocessor.pattern="{cmd.path}" "{source_file}" "{codecomplete}" -- -std=gnu++11
+
 # ctags
 # ------------------------------
 tools.ctags.path={runtime.tools.ctags.path}

preprocess_sketch.go

@@ -0,0 +1,120 @@
+/*
+ * This file is part of Arduino Builder.
+ *
+ * Arduino Builder is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As a special exception, you may use this file as part of a free software
+ * library without restriction.  Specifically, if other files instantiate
+ * templates or use macros or inline functions from this file, or you compile
+ * this file and link it with other files to produce an executable, this
+ * file does not by itself cause the resulting executable to be covered by
+ * the GNU General Public License.  This exception does not however
+ * invalidate any other reasons why the executable file might be covered by
+ * the GNU General Public License.
+ *
+ * Copyright 2015 Arduino LLC (http://www.arduino.cc/)
+ */
+
+package builder
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/arduino/arduino-builder/constants"
+	"github.com/arduino/arduino-builder/i18n"
+	"github.com/arduino/arduino-builder/types"
+	"github.com/arduino/arduino-builder/utils"
+)
+
+type PreprocessSketch struct{}
+
+func (s *PreprocessSketch) Run(ctx *types.Context) error {
+	sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp")
+	commands := []types.Command{
+		&GCCPreprocRunner{SourceFilePath: sourceFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders},
+		&ArduinoPreprocessorRunner{},
+	}
+
+	if ctx.CodeCompleteAt != "" {
+		commands = append(commands, &OutputCodeCompletions{})
+	} else {
+		commands = append(commands, &SketchSaver{})
+	}
+
+	for _, command := range commands {
+		PrintRingNameIfDebug(ctx, command)
+		err := command.Run(ctx)
+		if err != nil {
+			return i18n.WrapError(err)
+		}
+	}
+
+	return nil
+}
+
+type ArduinoPreprocessorRunner struct{}
+
+func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error {
+	buildProperties := ctx.BuildProperties
+	targetFilePath := ctx.FileToRead
+	logger := ctx.GetLogger()
+
+	properties := buildProperties.Clone()
+	toolProps := buildProperties.SubTree("tools").SubTree("arduino-preprocessor")
+	properties.Merge(toolProps)
+	properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = targetFilePath
+	if ctx.CodeCompleteAt != "" {
+		properties["codecomplete"] = "-output-code-completions=" + ctx.CodeCompleteAt
+	} else {
+		properties["codecomplete"] = ""
+	}
+
+	pattern := properties[constants.BUILD_PROPERTIES_PATTERN]
+	if pattern == constants.EMPTY_STRING {
+		return i18n.ErrorfWithLogger(logger, constants.MSG_PATTERN_MISSING, "arduino-preprocessor")
+	}
+
+	commandLine := properties.ExpandPropsInString(pattern)
+	command, err := utils.PrepareCommand(commandLine, logger)
+	if err != nil {
+		return i18n.WrapError(err)
+	}
+
+	verbose := ctx.Verbose
+	if verbose {
+		fmt.Println(commandLine)
+	}
+
+	buf, err := command.Output()
+	if err != nil {
+		return i18n.WrapError(err)
+	}
+	output := string(buf)
+	//fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", output)
+	if ctx.CodeCompleteAt != "" {
+		ctx.CodeCompletions = output
+	} else {
+		ctx.Source = output
+	}
+	return nil
+}
+
+type OutputCodeCompletions struct{}
+
+func (s *OutputCodeCompletions) Run(ctx *types.Context) error {
+	fmt.Println(ctx.CodeCompletions)
+	return nil
+}

test/helper_tools_downloader.go

@@ -112,7 +112,17 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) {
 				OsUrl{Os: "i686-mingw32", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-mingw32.zip"},
 				OsUrl{Os: "x86_64-apple-darwin", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-apple-darwin.zip"},
 				OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"},
-			}},
+			},
+		},
+		Tool{Name: "arduino-preprocessor", Version: "0.1.1",
+			OsUrls: []OsUrl{
+				OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-linux-gnu.tar.bz2"},
+				OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-x86_64-pc-linux-gnu.tar.bz2"},
+				OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-cygwin.tar.bz2"},
+				OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-x86_64-apple-darwin11.tar.bz2"},
+				OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-armhf-pc-linux-gnu.tar.bz2"},
+			},
+		},
 	}
 
 	boardsManagerTools := []Tool{
@@ -698,9 +708,9 @@ func translateGOOSGOARCHToPackageIndexValue() []string {
 	case "linux-386":
 		return []string{"i686-pc-linux-gnu", "i686-linux-gnu"}
 	case "windows-amd64":
-		return []string{"i686-mingw32"}
+		return []string{"i686-mingw32", "i686-cygwin"}
 	case "windows-386":
-		return []string{"i686-mingw32"}
+		return []string{"i686-mingw32", "i686-cygwin"}
 	case "darwin-amd64":
 		return []string{"i386-apple-darwin11", "x86_64-apple-darwin"}
 	case "linux-arm":

test/tools_loader_test.go

@@ -30,11 +30,12 @@
 package test
 
 import (
+	"sort"
+	"testing"
+
 	"github.com/arduino/arduino-builder"
 	"github.com/arduino/arduino-builder/types"
 	"github.com/stretchr/testify/require"
-	"sort"
-	"testing"
 )
 
 type ByToolIDAndVersion []*types.Tool
@@ -64,11 +65,15 @@ func TestLoadTools(t *testing.T) {
 	NoError(t, err)
 
 	tools := ctx.Tools
-	require.Equal(t, 6, len(tools))
+	require.Equal(t, 7, len(tools))
 
 	sort.Sort(ByToolIDAndVersion(tools))
 
 	idx := 0
+	require.Equal(t, "arduino-preprocessor", tools[idx].Name)
+	require.Equal(t, "0.1.1", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.1"), tools[idx].Folder)
+	idx++
 	require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name)
 	require.Equal(t, "4.8.3-2014q1", tools[idx].Version)
 	require.Equal(t, Abs(t, "./downloaded_tools/arm-none-eabi-gcc/4.8.3-2014q1"), tools[idx].Folder)
@@ -136,7 +141,7 @@ func TestLoadLotsOfTools(t *testing.T) {
 	NoError(t, err)
 
 	tools := ctx.Tools
-	require.Equal(t, 8, len(tools))
+	require.Equal(t, 9, len(tools))
 
 	sort.Sort(ByToolIDAndVersion(tools))
 
@@ -145,6 +150,10 @@ func TestLoadLotsOfTools(t *testing.T) {
 	require.Equal(t, "4.0.0-atmel", tools[idx].Version)
 	require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/CMSIS/4.0.0-atmel"), tools[idx].Folder)
 	idx++
+	require.Equal(t, "arduino-preprocessor", tools[idx].Name)
+	require.Equal(t, "0.1.1", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.1"), tools[idx].Folder)
+	idx++
 	require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name)
 	require.Equal(t, "4.8.3-2014q1", tools[idx].Version)
 	require.Equal(t, Abs(t, "./downloaded_tools/arm-none-eabi-gcc/4.8.3-2014q1"), tools[idx].Folder)

types/context.go

@@ -18,6 +18,7 @@ type Context struct {
 	SketchLocation          string
 	ArduinoAPIVersion       string
 	FQBN                    string
+	CodeCompleteAt          string
 
 	// Build options are serialized here
 	BuildOptionsJson         string
@@ -53,6 +54,7 @@ type Context struct {
 	Sketch          *Sketch
 	Source          string
 	SourceGccMinusE string
+	CodeCompletions string
 
 	WarningsLevel string