legacy: Arduino preprocess subroutine refactorization (part 1)

arduino/builder/cpp.go

@@ -0,0 +1,74 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package builder
+
+import (
+	"strings"
+	"unicode/utf8"
+)
+
+// QuoteCppString returns the given string as a quoted string for use with the C
+// preprocessor. This adds double quotes around it and escapes any
+// double quotes and backslashes in the string.
+func QuoteCppString(str string) string {
+	str = strings.Replace(str, "\\", "\\\\", -1)
+	str = strings.Replace(str, "\"", "\\\"", -1)
+	return "\"" + str + "\""
+}
+
+// ParseCppString parse a C-preprocessor string as emitted by the preprocessor. This
+// is a string contained in double quotes, with any backslashes or
+// quotes escaped with a backslash. If a valid string was present at the
+// start of the given line, returns the unquoted string contents, the
+// remainder of the line (everything after the closing "), and true.
+// Otherwise, returns the empty string, the entire line and false.
+func ParseCppString(line string) (string, string, bool) {
+	// For details about how these strings are output by gcc, see:
+	// https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511
+	// Note that the documentation suggests all non-printable
+	// characters are also escaped, but the implementation does not
+	// actually do this. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51259
+	if len(line) < 1 || line[0] != '"' {
+		return "", line, false
+	}
+
+	i := 1
+	res := ""
+	for {
+		if i >= len(line) {
+			return "", line, false
+		}
+
+		c, width := utf8.DecodeRuneInString(line[i:])
+
+		switch c {
+		case '\\':
+			// Backslash, next character is used unmodified
+			i += width
+			if i >= len(line) {
+				return "", line, false
+			}
+			res += string(line[i])
+		case '"':
+			// Quote, end of string
+			return res, line[i+width:], true
+		default:
+			res += string(c)
+		}
+
+		i += width
+	}
+}

arduino/builder/cpp_test.go

@@ -0,0 +1,46 @@
+package builder_test
+
+import (
+	"testing"
+
+	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseCppString(t *testing.T) {
+	_, _, ok := builder.ParseCppString(`foo`)
+	require.Equal(t, false, ok)
+
+	_, _, ok = builder.ParseCppString(`"foo`)
+	require.Equal(t, false, ok)
+
+	str, rest, ok := builder.ParseCppString(`"foo"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"foo\\bar"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo\bar`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str)
+	require.Equal(t, ` and "then" some`, rest)
+
+	str, rest, ok = builder.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"/home/ççç/"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `/home/ççç/`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"/home/ççç/ /$sdsdd\\"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `/home/ççç/ /$sdsdd\`, str)
+	require.Equal(t, ``, rest)
+}

arduino/builder/sketch.go

@@ -19,7 +19,6 @@ import (
 	"bytes"
 	"fmt"
 	"regexp"
-	"strings"
 
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/i18n"
@@ -34,13 +33,20 @@ var (
 	tr               = i18n.Tr
 )
 
-// QuoteCppString returns the given string as a quoted string for use with the C
-// preprocessor. This adds double quotes around it and escapes any
-// double quotes and backslashes in the string.
-func QuoteCppString(str string) string {
-	str = strings.Replace(str, "\\", "\\\\", -1)
-	str = strings.Replace(str, "\"", "\\\"", -1)
-	return "\"" + str + "\""
+// PrepareSketchBuildPath copies the sketch source files in the build path.
+// The .ino files are merged together to create a .cpp file (by the way, the
+// .cpp file still needs to be Arduino-preprocessed to compile).
+func PrepareSketchBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (offset int, mergedSource string, err error) {
+	if offset, mergedSource, err = sketchMergeSources(sketch, sourceOverrides); err != nil {
+		return
+	}
+	if err = SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil {
+		return
+	}
+	if err = sketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil {
+		return
+	}
+	return
 }
 
 // SketchSaveItemCpp saves a preprocessed .cpp sketch file on disk
@@ -59,8 +65,9 @@ func SketchSaveItemCpp(path *paths.Path, contents []byte, destPath *paths.Path)
 	return nil
 }
 
-// SketchMergeSources merges all the source files included in a sketch
-func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) {
+// sketchMergeSources merges all the .ino source files included in a sketch to produce
+// a single .cpp file.
+func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) {
 	lineOffset := 0
 	mergedSource := ""
 
@@ -105,9 +112,9 @@ func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st
 	return lineOffset, mergedSource, nil
 }
 
-// SketchCopyAdditionalFiles copies the additional files for a sketch to the
+// sketchCopyAdditionalFiles copies the additional files for a sketch to the
 // specified destination directory.
-func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error {
+func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error {
 	if err := destPath.MkdirAll(); err != nil {
 		return errors.Wrap(err, tr("unable to create a folder to save the sketch files"))
 	}

arduino/builder/sketch_test.go

@@ -13,7 +13,7 @@
 // Arduino software without disclosing the source code of your own applications.
 // To purchase a commercial license, send an email to license@arduino.cc.
 
-package builder_test
+package builder
 
 import (
 	"fmt"
@@ -23,7 +23,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
@@ -48,7 +47,7 @@ func TestSaveSketch(t *testing.T) {
 		t.Fatalf("unable to read golden file %s: %v", sketchFile, err)
 	}
 
-	builder.SketchSaveItemCpp(paths.New(sketchName), source, tmp)
+	SketchSaveItemCpp(paths.New(sketchName), source, tmp)
 
 	out, err := tmp.Join(outName).ReadFile()
 	if err != nil {
@@ -82,7 +81,7 @@ func TestMergeSketchSources(t *testing.T) {
 	}
 	mergedSources := strings.ReplaceAll(string(mergedBytes), "%s", pathToGoldenSource)
 
-	offset, source, err := builder.SketchMergeSources(s, nil)
+	offset, source, err := sketchMergeSources(s, nil)
 	require.Nil(t, err)
 	require.Equal(t, 2, offset)
 	require.Equal(t, mergedSources, source)
@@ -94,7 +93,7 @@ func TestMergeSketchSourcesArduinoIncluded(t *testing.T) {
 	require.NotNil(t, s)
 
 	// ensure not to include Arduino.h when it's already there
-	_, source, err := builder.SketchMergeSources(s, nil)
+	_, source, err := sketchMergeSources(s, nil)
 	require.Nil(t, err)
 	require.Equal(t, 1, strings.Count(source, "<Arduino.h>"))
 }
@@ -110,7 +109,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
 
 	// copy the sketch over, create a fake main file we don't care about it
 	// but we need it for `SketchLoad` to succeed later
-	err = builder.SketchCopyAdditionalFiles(s1, tmp, nil)
+	err = sketchCopyAdditionalFiles(s1, tmp, nil)
 	require.Nil(t, err)
 	fakeIno := tmp.Join(fmt.Sprintf("%s.ino", tmp.Base()))
 	require.Nil(t, fakeIno.WriteFile([]byte{}))
@@ -125,7 +124,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
 	require.Nil(t, err)
 
 	// copy again
-	err = builder.SketchCopyAdditionalFiles(s1, tmp, nil)
+	err = sketchCopyAdditionalFiles(s1, tmp, nil)
 	require.Nil(t, err)
 
 	// verify file hasn't changed

legacy/builder/builder.go

@@ -19,6 +19,7 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/phases"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -39,14 +40,18 @@ func (s *Builder) Run(ctx *types.Context) error {
 		return err
 	}
 
+	var _err error
 	commands := []types.Command{
 		&ContainerSetupHardwareToolsLibsSketchAndProps{},
 
 		&ContainerBuildOptions{},
 
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"},
 
-		&ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 
 		utils.LogIfVerbose(false, tr("Detecting libraries used...")),
 		&ContainerFindIncludes{},
@@ -127,14 +132,18 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 		return err
 	}
 
+	var _err error
 	commands := []types.Command{
 		&ContainerSetupHardwareToolsLibsSketchAndProps{},
 
 		&ContainerBuildOptions{},
 
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"},
 
-		&ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 
 		&ContainerFindIncludes{},
 
@@ -148,7 +157,7 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 	}
 
 	// Output arduino-preprocessed source
-	ctx.WriteStdout([]byte(ctx.Source))
+	ctx.WriteStdout([]byte(ctx.SketchSourceAfterArduinoPreprocessing))
 	return nil
 }