Merge remote-tracking branch 'upstream/master'

Author: robertinant

README.md

@@ -69,6 +69,7 @@ go get google.golang.org/grpc
 go get github.com/golang/protobuf/proto
 go get golang.org/x/net/context
 go get github.com/fsnotify/fsnotify
+go get github.com/schollz/closestmatch
 go get github.com/arduino/arduino-builder
 go build github.com/arduino/arduino-builder/arduino-builder
 ```

arduino-builder/main.go

@@ -37,6 +37,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"syscall"
@@ -54,7 +55,7 @@ import (
 	"github.com/go-errors/errors"
 )
 
-const VERSION = "1.4.1"
+const VERSION = "1.4.4"
 
 const FLAG_ACTION_COMPILE = "compile"
 const FLAG_ACTION_PREPROCESS = "preprocess"
@@ -96,19 +97,8 @@ func (h *foldersFlag) String() string {
 	return fmt.Sprint(*h)
 }
 
-func (h *foldersFlag) Set(csv string) error {
-	var values []string
-	if strings.Contains(csv, string(os.PathListSeparator)) {
-		values = strings.Split(csv, string(os.PathListSeparator))
-	} else {
-		values = strings.Split(csv, ",")
-	}
-
-	for _, value := range values {
-		value = strings.TrimSpace(value)
-		*h = append(*h, value)
-	}
-
+func (h *foldersFlag) Set(folder string) error {
+	*h = append(*h, folder)
 	return nil
 }
 
@@ -321,8 +311,8 @@ func main() {
 		if err != nil {
 			printCompleteError(err)
 		}
+		ctx.BuildPath, _ = filepath.Abs(buildPath)
 	}
-	ctx.BuildPath = buildPath
 
 	// FLAG_BUILD_CACHE
 	buildCachePath, err := gohasissues.Unquote(*buildCachePathFlag)

create_cmake_rule.go

@@ -35,14 +35,16 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/arduino/go-properties-map"
+
 	"github.com/arduino/arduino-builder/builder_utils"
 	"github.com/arduino/arduino-builder/constants"
 	"github.com/arduino/arduino-builder/i18n"
 	"github.com/arduino/arduino-builder/types"
 	"github.com/arduino/arduino-builder/utils"
 )
 
-var VALID_EXPORT_EXTENSIONS = map[string]bool{".h": true, ".c": true, ".hpp": true, ".hh": true, ".cpp": true, ".s": true, ".a": true}
+var VALID_EXPORT_EXTENSIONS = map[string]bool{".h": true, ".c": true, ".hpp": true, ".hh": true, ".cpp": true, ".s": true, ".a": true, ".properties": true}
 var DOTHEXTENSION = map[string]bool{".h": true, ".hh": true, ".hpp": true}
 var DOTAEXTENSION = map[string]bool{".a": true}
 
@@ -76,23 +78,39 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 	coreFolder := filepath.Join(cmakeFolder, "core")
 	cmakeFile := filepath.Join(cmakeFolder, "CMakeLists.txt")
 
-	// Copy used  libraries in the correct folder
+	dynamicLibsFromPkgConfig := map[string]bool{}
 	extensions := func(ext string) bool { return VALID_EXPORT_EXTENSIONS[ext] }
+	staticLibsExtensions := func(ext string) bool { return DOTAEXTENSION[ext] }
 	for _, library := range ctx.ImportedLibraries {
+		// Copy used libraries in the correct folder
 		libFolder := filepath.Join(libBaseFolder, library.Name)
+		mcu := ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_MCU]
 		utils.CopyDir(library.Folder, libFolder, extensions)
+
+		// Read cmake options if available
+		isStaticLib := true
+		if cmakeOptions, err := properties.Load(filepath.Join(libFolder, "src", mcu, "arduino_builder.properties")); err == nil {
+			// If the library can be linked dynamically do not copy the library folder
+			if pkgs, ok := cmakeOptions["cmake.pkg_config"]; ok {
+				isStaticLib = false
+				for _, pkg := range strings.Split(pkgs, " ") {
+					dynamicLibsFromPkgConfig[pkg] = true
+				}
+			}
+		}
+
 		// Remove examples folder
 		if _, err := os.Stat(filepath.Join(libFolder, "examples")); err == nil {
 			os.RemoveAll(filepath.Join(libFolder, "examples"))
 		}
-		// Remove stray folders contining incompatible libraries
-		staticLibsExtensions := func(ext string) bool { return DOTAEXTENSION[ext] }
-		mcu := ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_MCU]
+
+		// Remove stray folders contining incompatible or not needed libraries archives
 		var files []string
 		utils.FindFilesInFolder(&files, filepath.Join(libFolder, "src"), staticLibsExtensions, true)
 		for _, file := range files {
-			if !strings.Contains(filepath.Dir(file), mcu) {
-				os.RemoveAll(filepath.Dir(file))
+			staticLibDir := filepath.Dir(file)
+			if !isStaticLib || !strings.Contains(staticLibDir, mcu) {
+				os.RemoveAll(staticLibDir)
 			}
 		}
 	}
@@ -109,10 +127,10 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 
 	// Use old ctags method to generate export file
 	commands := []types.Command{
-		&ContainerMergeCopySketchFiles{},
+		//&ContainerMergeCopySketchFiles{},
 		&ContainerAddPrototypes{},
-		&FilterSketchSource{Source: &ctx.Source, RemoveLineMarkers: true},
-		&SketchSaver{},
+		//&FilterSketchSource{Source: &ctx.Source, RemoveLineMarkers: true},
+		//&SketchSaver{},
 	}
 
 	for _, command := range commands {
@@ -127,12 +145,12 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 	// Extract CFLAGS, CPPFLAGS and LDFLAGS
 	var defines []string
 	var linkerflags []string
-	var libs []string
+	var dynamicLibsFromGccMinusL []string
 	var linkDirectories []string
 
-	extractCompileFlags(ctx, constants.RECIPE_C_COMBINE_PATTERN, &defines, &libs, &linkerflags, &linkDirectories, logger)
-	extractCompileFlags(ctx, constants.RECIPE_C_PATTERN, &defines, &libs, &linkerflags, &linkDirectories, logger)
-	extractCompileFlags(ctx, constants.RECIPE_CPP_PATTERN, &defines, &libs, &linkerflags, &linkDirectories, logger)
+	extractCompileFlags(ctx, constants.RECIPE_C_COMBINE_PATTERN, &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories, logger)
+	extractCompileFlags(ctx, constants.RECIPE_C_PATTERN, &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories, logger)
+	extractCompileFlags(ctx, constants.RECIPE_CPP_PATTERN, &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories, logger)
 
 	// Extract folders with .h in them for adding in include list
 	var headerFiles []string
@@ -141,9 +159,8 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 	foldersContainingDotH := findUniqueFoldersRelative(headerFiles, cmakeFolder)
 
 	// Extract folders with .a in them for adding in static libs paths list
-	var staticLibsFiles []string
-	isStaticLib := func(ext string) bool { return DOTAEXTENSION[ext] }
-	utils.FindFilesInFolder(&staticLibsFiles, cmakeFolder, isStaticLib, true)
+	var staticLibs []string
+	utils.FindFilesInFolder(&staticLibs, cmakeFolder, staticLibsExtensions, true)
 
 	// Generate the CMakeLists global file
 
@@ -168,33 +185,39 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 	// Add SO_PATHS option for libraries not getting found by pkg_config
 	cmakelist += "set(EXTRA_LIBS_DIRS \"\" CACHE STRING \"Additional paths for dynamic libraries\")\n"
 
-	for i, lib := range libs {
+	linkGroup := ""
+	for _, lib := range dynamicLibsFromGccMinusL {
 		// Dynamic libraries should be discovered by pkg_config
-		lib = strings.TrimPrefix(lib, "-l")
-		libs[i] = lib
 		cmakelist += "pkg_search_module (" + strings.ToUpper(lib) + " " + lib + ")\n"
 		relLinkDirectories = append(relLinkDirectories, "${"+strings.ToUpper(lib)+"_LIBRARY_DIRS}")
+		linkGroup += " " + lib
+	}
+	for lib := range dynamicLibsFromPkgConfig {
+		cmakelist += "pkg_search_module (" + strings.ToUpper(lib) + " " + lib + ")\n"
+		relLinkDirectories = append(relLinkDirectories, "${"+strings.ToUpper(lib)+"_LIBRARY_DIRS}")
+		linkGroup += " ${" + strings.ToUpper(lib) + "_LIBRARIES}"
 	}
 	cmakelist += "link_directories (" + strings.Join(relLinkDirectories, " ") + " ${EXTRA_LIBS_DIRS})\n"
-	for _, staticLibsFile := range staticLibsFiles {
+	for _, staticLib := range staticLibs {
 		// Static libraries are fully configured
-		lib := filepath.Base(staticLibsFile)
+		lib := filepath.Base(staticLib)
 		lib = strings.TrimPrefix(lib, "lib")
 		lib = strings.TrimSuffix(lib, ".a")
-		if !utils.SliceContains(libs, lib) {
-			libs = append(libs, lib)
+		if !utils.SliceContains(dynamicLibsFromGccMinusL, lib) {
+			linkGroup += " " + lib
 			cmakelist += "add_library (" + lib + " STATIC IMPORTED)\n"
-			location := strings.TrimPrefix(staticLibsFile, cmakeFolder)
+			location := strings.TrimPrefix(staticLib, cmakeFolder)
 			cmakelist += "set_property(TARGET " + lib + " PROPERTY IMPORTED_LOCATION " + "${PROJECT_SOURCE_DIR}" + location + " )\n"
 		}
 	}
+
 	// Include source files
 	// TODO: remove .cpp and .h from libraries example folders
 	cmakelist += "file (GLOB_RECURSE SOURCES core/*.c* lib/*.c* sketch/*.c*)\n"
 
 	// Compile and link project
 	cmakelist += "add_executable (" + projectName + " ${SOURCES} ${SOURCES_LIBS})\n"
-	cmakelist += "target_link_libraries( " + projectName + " -Wl,--as-needed -Wl,--start-group " + strings.Join(libs, " ") + " -Wl,--end-group)\n"
+	cmakelist += "target_link_libraries( " + projectName + " -Wl,--as-needed -Wl,--start-group " + linkGroup + " -Wl,--end-group)\n"
 
 	utils.WriteFile(cmakeFile, cmakelist)
 
@@ -205,26 +228,26 @@ func canExportCmakeProject(ctx *types.Context) bool {
 	return ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_EXPORT_CMAKE_FLAGS] != ""
 }
 
-func extractCompileFlags(ctx *types.Context, receipe string, defines, libs, linkerflags, linkDirectories *[]string, logger i18n.Logger) {
+func extractCompileFlags(ctx *types.Context, receipe string, defines, dynamicLibs, linkerflags, linkDirectories *[]string, logger i18n.Logger) {
 	command, _ := builder_utils.PrepareCommandForRecipe(ctx, ctx.BuildProperties, receipe, true)
 
 	for _, arg := range command.Args {
 		if strings.HasPrefix(arg, "-D") {
-			*defines = appendIfUnique(*defines, arg)
+			*defines = utils.AppendIfNotPresent(*defines, arg)
 			continue
 		}
 		if strings.HasPrefix(arg, "-l") {
-			*libs = appendIfUnique(*libs, arg)
+			*dynamicLibs = utils.AppendIfNotPresent(*dynamicLibs, arg[2:])
 			continue
 		}
 		if strings.HasPrefix(arg, "-L") {
-			*linkDirectories = appendIfUnique(*linkDirectories, strings.TrimPrefix(arg, "-L"))
+			*linkDirectories = utils.AppendIfNotPresent(*linkDirectories, arg[2:])
 			continue
 		}
 		if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "-I") && !strings.HasPrefix(arg, "-o") {
 			// HACK : from linkerflags remove MMD (no cache is produced)
 			if !strings.HasPrefix(arg, "-MMD") {
-				*linkerflags = appendIfUnique(*linkerflags, arg)
+				*linkerflags = utils.AppendIfNotPresent(*linkerflags, arg)
 			}
 		}
 	}
@@ -241,10 +264,3 @@ func findUniqueFoldersRelative(slice []string, base string) string {
 	}
 	return strings.Join(out, " ")
 }
-
-func appendIfUnique(slice []string, element string) []string {
-	if !utils.SliceContains(slice, element) {
-		slice = append(slice, element)
-	}
-	return slice
-}

ctags/ctags_to_prototypes.go

@@ -72,8 +72,7 @@ func functionNameUsedAsFunctionPointerIn(tag *types.CTag, functionTags []*types.
 		if tag.Line != functionTag.Line && strings.Index(tag.Code, "&"+functionTag.FunctionName) != -1 {
 			return true
 		}
-		if tag.Line != functionTag.Line && strings.Index(tag.Code, functionTag.FunctionName) != -1 &&
-			(functionTag.Signature == "(void)" || functionTag.Signature == "()") {
+		if tag.Line != functionTag.Line && strings.Index(strings.TrimSpace(tag.Code), "("+functionTag.FunctionName+")") != -1 {
 			return true
 		}
 	}

resolve_library.go

@@ -36,6 +36,7 @@ import (
 	"github.com/arduino/arduino-builder/constants"
 	"github.com/arduino/arduino-builder/types"
 	"github.com/arduino/arduino-builder/utils"
+	"github.com/schollz/closestmatch"
 )
 
 func ResolveLibrary(ctx *types.Context, header string) *types.Library {
@@ -211,6 +212,10 @@ func findBestLibraryWithHeader(header string, libraries []*types.Library) *types
 		if library != nil {
 			return library
 		}
+		library = findLibWithNameBestDistance(headerName, libraries)
+		if library != nil {
+			return library
+		}
 	}
 
 	return nil
@@ -252,6 +257,28 @@ func findLibWithNameContaining(name string, libraries []*types.Library) *types.L
 	return nil
 }
 
+func findLibWithNameBestDistance(name string, libraries []*types.Library) *types.Library {
+	// create closestmatch DB
+	var wordsToTest []string
+
+	for _, library := range libraries {
+			wordsToTest = append(wordsToTest, simplifyName(library.Name))
+	}
+	// Choose a set of bag sizes, more is more accurate but slower
+	bagSizes := []int{2}
+	// Create a closestmatch object
+	cm := closestmatch.New(wordsToTest, bagSizes)
+	closest_name := cm.Closest(name)
+
+	for _, library := range libraries {
+		if (closest_name == simplifyName(library.Name)) {
+			return library
+		}
+	}
+
+	return nil
+}
+
 func simplifyName(name string) string {
 	return strings.ToLower(strings.Replace(name, "_", " ", -1))
 }

resolve_library_test.go

@@ -43,29 +43,38 @@ func TestFindBestLibraryWithHeader(t *testing.T) {
 	l3 := &types.Library{Name: "Calculus Lib Improved"}
 	l4 := &types.Library{Name: "Another Calculus Lib"}
 	l5 := &types.Library{Name: "Yet Another Calculus Lib Improved"}
-	l6 := &types.Library{Name: "AnotherLib"}
+	l6 := &types.Library{Name: "Calculus Unified Lib"}
+	l7 := &types.Library{Name: "AnotherLib"}
 
 	// Test exact name matching
-	res := findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l6, l5, l4, l3, l2, l1})
+	res := findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l7, l6, l5, l4, l3, l2, l1})
 	require.Equal(t, l1.Name, res.Name)
 
 	// Test exact name with "-master" postfix matching
-	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l6, l5, l4, l3, l2})
+	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l7, l6, l5, l4, l3, l2})
 	require.Equal(t, l2.Name, res.Name)
 
 	// Test prefix matching
-	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l6, l5, l4, l3})
+	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l7, l6, l5, l4, l3})
 	require.Equal(t, l3.Name, res.Name)
 
 	// Test postfix matching
-	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l6, l5, l4})
+	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l7, l6, l5, l4})
 	require.Equal(t, l4.Name, res.Name)
 
 	// Test "contains"" matching
-	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l6, l5})
+	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l7, l6, l5})
 	require.Equal(t, l5.Name, res.Name)
 
+	// Test lexicographic similarity matching
+	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l7, l6})
+	require.Equal(t, l6.Name, res.Name)
+
+	// Test lexicographic similarity matching (2)
+	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l6, l7})
+	require.Equal(t, l6.Name, res.Name)
+
 	// Test none matching
-	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l6})
-	require.Nil(t, res)
+	res = findBestLibraryWithHeader("calculus_lib.h", []*types.Library{l7})
+	require.Equal(t, l7.Name, res.Name)
 }

sketch_loader.go

@@ -76,7 +76,7 @@ func (s *SketchLoader) Run(ctx *types.Context) error {
 		return i18n.ErrorfWithLogger(logger, constants.MSG_CANT_FIND_SKETCH_IN_PATH, sketchLocation, filepath.Dir(sketchLocation))
 	}
 
-	sketch, err := makeSketch(sketchLocation, allSketchFilePaths, logger)
+	sketch, err := makeSketch(sketchLocation, allSketchFilePaths, ctx.BuildPath, logger)
 	if err != nil {
 		return i18n.WrapError(err)
 	}
@@ -100,7 +100,7 @@ func collectAllSketchFiles(from string) ([]string, error) {
 	return filePaths, i18n.WrapError(err)
 }
 
-func makeSketch(sketchLocation string, allSketchFilePaths []string, logger i18n.Logger) (*types.Sketch, error) {
+func makeSketch(sketchLocation string, allSketchFilePaths []string, buildLocation string, logger i18n.Logger) (*types.Sketch, error) {
 	sketchFilesMap := make(map[string]types.SketchFile)
 	for _, sketchFilePath := range allSketchFilePaths {
 		source, err := ioutil.ReadFile(sketchFilePath)
@@ -123,7 +123,9 @@ func makeSketch(sketchLocation string, allSketchFilePaths []string, logger i18n.
 				otherSketchFiles = append(otherSketchFiles, sketchFile)
 			}
 		} else if ADDITIONAL_FILE_VALID_EXTENSIONS[ext] {
-			additionalFiles = append(additionalFiles, sketchFile)
+			if !strings.Contains(filepath.Dir(sketchFile.Name), buildLocation) || buildLocation == "" {
+				additionalFiles = append(additionalFiles, sketchFile)
+			}
 		} else {
 			return nil, i18n.ErrorfWithLogger(logger, constants.MSG_UNKNOWN_SKETCH_EXT, sketchFile.Name)
 		}