diff --git a/arduino/builder/cpp.go b/arduino/builder/cpp.go new file mode 100644 index 00000000000..87610e7ed18 --- /dev/null +++ b/arduino/builder/cpp.go @@ -0,0 +1,74 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package builder + +import ( + "strings" + "unicode/utf8" +) + +// QuoteCppString returns the given string as a quoted string for use with the C +// preprocessor. This adds double quotes around it and escapes any +// double quotes and backslashes in the string. +func QuoteCppString(str string) string { + str = strings.Replace(str, "\\", "\\\\", -1) + str = strings.Replace(str, "\"", "\\\"", -1) + return "\"" + str + "\"" +} + +// ParseCppString parse a C-preprocessor string as emitted by the preprocessor. This +// is a string contained in double quotes, with any backslashes or +// quotes escaped with a backslash. If a valid string was present at the +// start of the given line, returns the unquoted string contents, the +// remainder of the line (everything after the closing "), and true. +// Otherwise, returns the empty string, the entire line and false. +func ParseCppString(line string) (string, string, bool) { + // For details about how these strings are output by gcc, see: + // https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511 + // Note that the documentation suggests all non-printable + // characters are also escaped, but the implementation does not + // actually do this. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51259 + if len(line) < 1 || line[0] != '"' { + return "", line, false + } + + i := 1 + res := "" + for { + if i >= len(line) { + return "", line, false + } + + c, width := utf8.DecodeRuneInString(line[i:]) + + switch c { + case '\\': + // Backslash, next character is used unmodified + i += width + if i >= len(line) { + return "", line, false + } + res += string(line[i]) + case '"': + // Quote, end of string + return res, line[i+width:], true + default: + res += string(c) + } + + i += width + } +} diff --git a/arduino/builder/cpp_test.go b/arduino/builder/cpp_test.go new file mode 100644 index 00000000000..84de5f0b254 --- /dev/null +++ b/arduino/builder/cpp_test.go @@ -0,0 +1,46 @@ +package builder_test + +import ( + "testing" + + "github.com/arduino/arduino-cli/arduino/builder" + "github.com/stretchr/testify/require" +) + +func TestParseCppString(t *testing.T) { + _, _, ok := builder.ParseCppString(`foo`) + require.Equal(t, false, ok) + + _, _, ok = builder.ParseCppString(`"foo`) + require.Equal(t, false, ok) + + str, rest, ok := builder.ParseCppString(`"foo"`) + require.Equal(t, true, ok) + require.Equal(t, `foo`, str) + require.Equal(t, ``, rest) + + str, rest, ok = builder.ParseCppString(`"foo\\bar"`) + require.Equal(t, true, ok) + require.Equal(t, `foo\bar`, str) + require.Equal(t, ``, rest) + + str, rest, ok = builder.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`) + require.Equal(t, true, ok) + require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str) + require.Equal(t, ` and "then" some`, rest) + + str, rest, ok = builder.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`) + require.Equal(t, true, ok) + require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str) + require.Equal(t, ``, rest) + + str, rest, ok = builder.ParseCppString(`"/home/ççç/"`) + require.Equal(t, true, ok) + require.Equal(t, `/home/ççç/`, str) + require.Equal(t, ``, rest) + + str, rest, ok = builder.ParseCppString(`"/home/ççç/ /$sdsdd\\"`) + require.Equal(t, true, ok) + require.Equal(t, `/home/ççç/ /$sdsdd\`, str) + require.Equal(t, ``, rest) +} diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index b5a6a97abb9..8560178a193 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -19,7 +19,6 @@ import ( "bytes" "fmt" "regexp" - "strings" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/i18n" @@ -34,13 +33,20 @@ var ( tr = i18n.Tr ) -// QuoteCppString returns the given string as a quoted string for use with the C -// preprocessor. This adds double quotes around it and escapes any -// double quotes and backslashes in the string. -func QuoteCppString(str string) string { - str = strings.Replace(str, "\\", "\\\\", -1) - str = strings.Replace(str, "\"", "\\\"", -1) - return "\"" + str + "\"" +// PrepareSketchBuildPath copies the sketch source files in the build path. +// The .ino files are merged together to create a .cpp file (by the way, the +// .cpp file still needs to be Arduino-preprocessed to compile). +func PrepareSketchBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (offset int, mergedSource string, err error) { + if offset, mergedSource, err = sketchMergeSources(sketch, sourceOverrides); err != nil { + return + } + if err = SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil { + return + } + if err = sketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil { + return + } + return } // SketchSaveItemCpp saves a preprocessed .cpp sketch file on disk @@ -59,8 +65,9 @@ func SketchSaveItemCpp(path *paths.Path, contents []byte, destPath *paths.Path) return nil } -// SketchMergeSources merges all the source files included in a sketch -func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) { +// sketchMergeSources merges all the .ino source files included in a sketch to produce +// a single .cpp file. +func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) { lineOffset := 0 mergedSource := "" @@ -105,9 +112,9 @@ func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st return lineOffset, mergedSource, nil } -// SketchCopyAdditionalFiles copies the additional files for a sketch to the +// sketchCopyAdditionalFiles copies the additional files for a sketch to the // specified destination directory. -func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error { +func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error { if err := destPath.MkdirAll(); err != nil { return errors.Wrap(err, tr("unable to create a folder to save the sketch files")) } diff --git a/arduino/builder/sketch_test.go b/arduino/builder/sketch_test.go index a8f1cb6b1b0..5466cdff819 100644 --- a/arduino/builder/sketch_test.go +++ b/arduino/builder/sketch_test.go @@ -13,7 +13,7 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package builder_test +package builder import ( "fmt" @@ -23,7 +23,6 @@ import ( "strings" "testing" - "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" @@ -48,7 +47,7 @@ func TestSaveSketch(t *testing.T) { t.Fatalf("unable to read golden file %s: %v", sketchFile, err) } - builder.SketchSaveItemCpp(paths.New(sketchName), source, tmp) + SketchSaveItemCpp(paths.New(sketchName), source, tmp) out, err := tmp.Join(outName).ReadFile() if err != nil { @@ -82,7 +81,7 @@ func TestMergeSketchSources(t *testing.T) { } mergedSources := strings.ReplaceAll(string(mergedBytes), "%s", pathToGoldenSource) - offset, source, err := builder.SketchMergeSources(s, nil) + offset, source, err := sketchMergeSources(s, nil) require.Nil(t, err) require.Equal(t, 2, offset) require.Equal(t, mergedSources, source) @@ -94,7 +93,7 @@ func TestMergeSketchSourcesArduinoIncluded(t *testing.T) { require.NotNil(t, s) // ensure not to include Arduino.h when it's already there - _, source, err := builder.SketchMergeSources(s, nil) + _, source, err := sketchMergeSources(s, nil) require.Nil(t, err) require.Equal(t, 1, strings.Count(source, "")) } @@ -110,7 +109,7 @@ func TestCopyAdditionalFiles(t *testing.T) { // copy the sketch over, create a fake main file we don't care about it // but we need it for `SketchLoad` to succeed later - err = builder.SketchCopyAdditionalFiles(s1, tmp, nil) + err = sketchCopyAdditionalFiles(s1, tmp, nil) require.Nil(t, err) fakeIno := tmp.Join(fmt.Sprintf("%s.ino", tmp.Base())) require.Nil(t, fakeIno.WriteFile([]byte{})) @@ -125,7 +124,7 @@ func TestCopyAdditionalFiles(t *testing.T) { require.Nil(t, err) // copy again - err = builder.SketchCopyAdditionalFiles(s1, tmp, nil) + err = sketchCopyAdditionalFiles(s1, tmp, nil) require.Nil(t, err) // verify file hasn't changed diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index d73ec778ca7..b1b1f007c5d 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -19,6 +19,7 @@ import ( "reflect" "time" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/phases" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -39,6 +40,7 @@ func (s *Builder) Run(ctx *types.Context) error { return err } + var _err error commands := []types.Command{ &ContainerSetupHardwareToolsLibsSketchAndProps{}, @@ -46,7 +48,10 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, - &ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), utils.LogIfVerbose(false, tr("Detecting libraries used...")), &ContainerFindIncludes{}, @@ -127,6 +132,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { return err } + var _err error commands := []types.Command{ &ContainerSetupHardwareToolsLibsSketchAndProps{}, @@ -134,7 +140,10 @@ func (s *Preprocess) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, - &ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &ContainerFindIncludes{}, @@ -148,7 +157,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { } // Output arduino-preprocessed source - ctx.WriteStdout([]byte(ctx.Source)) + ctx.WriteStdout([]byte(ctx.SketchSourceAfterArduinoPreprocessing)) return nil } diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index d710b82c979..fe5e9064742 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -16,10 +16,17 @@ package builder import ( + "bufio" + "bytes" "fmt" + "io" + "strconv" + "strings" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/legacy/builder/types" + "github.com/arduino/go-paths-helper" "github.com/pkg/errors" ) @@ -28,7 +35,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { if err := ctx.PreprocPath.MkdirAll(); err != nil { return errors.WithStack(err) } - targetFilePath := ctx.PreprocPath.Join("ctags_target_for_gcc_minus_e.cpp") + targetFilePath := ctx.PreprocPath.Join("sketch_merged.cpp") // Run preprocessor sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") @@ -47,11 +54,14 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { } } + if src, err := targetFilePath.ReadFile(); err != nil { + return err + } else { + ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false) + } + commands := []types.Command{ - &ReadFileAndStoreInContext{FileToRead: targetFilePath, Target: &ctx.SourceGccMinusE}, - &FilterSketchSource{Source: &ctx.SourceGccMinusE}, - &CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: "ctags_target_for_gcc_minus_e.cpp"}, - &CTagsRunner{}, + &CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}, &PrototypesAdder{}, } @@ -63,9 +73,66 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { } } - if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.Source), ctx.SketchBuildPath); err != nil { + if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil { return errors.WithStack(err) } return nil } + +func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string { + fileNames := paths.NewPathList() + fileNames.Add(sketch.MainFile) + fileNames.AddAll(sketch.OtherSketchFiles) + + inSketch := false + filtered := "" + + scanner := bufio.NewScanner(source) + for scanner.Scan() { + line := scanner.Text() + if filename := parseLineMarker(line); filename != nil { + inSketch = fileNames.Contains(filename) + if inSketch && removeLineMarkers { + continue + } + } + + if inSketch { + filtered += line + "\n" + } + } + + return filtered +} + +// Parses the given line as a gcc line marker and returns the contained +// filename. +func parseLineMarker(line string) *paths.Path { + // A line marker contains the line number and filename and looks like: + // # 123 /path/to/file.cpp + // It can be followed by zero or more flag number that indicate the + // preprocessor state and can be ignored. + // For exact details on this format, see: + // https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415 + + split := strings.SplitN(line, " ", 3) + if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' { + return nil + } + + _, err := strconv.Atoi(split[1]) + if err != nil { + return nil + } + + // If we get here, we found a # followed by a line number, so + // assume this is a line marker and see if the rest of the line + // starts with a string containing the filename + str, rest, ok := bldr.ParseCppString(split[2]) + + if ok && (rest == "" || rest[0] == ' ') { + return paths.New(str) + } + return nil +} diff --git a/legacy/builder/container_merge_copy_sketch_files.go b/legacy/builder/container_merge_copy_sketch_files.go deleted file mode 100644 index b98eaebb721..00000000000 --- a/legacy/builder/container_merge_copy_sketch_files.go +++ /dev/null @@ -1,43 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - bldr "github.com/arduino/arduino-cli/arduino/builder" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" -) - -type ContainerMergeCopySketchFiles struct{} - -func (s *ContainerMergeCopySketchFiles) Run(ctx *types.Context) error { - offset, source, err := bldr.SketchMergeSources(ctx.Sketch, ctx.SourceOverride) - if err != nil { - return err - } - ctx.LineOffset = offset - ctx.Source = source - - if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.Source), ctx.SketchBuildPath); err != nil { - return errors.WithStack(err) - } - - if err := bldr.SketchCopyAdditionalFiles(ctx.Sketch, ctx.SketchBuildPath, ctx.SourceOverride); err != nil { - return errors.WithStack(err) - } - - return nil -} diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 16a9aa7781a..03549563780 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -120,12 +120,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { } // Use old ctags method to generate export file - commands := []types.Command{ - &FilterSketchSource{Source: &ctx.Source, RemoveLineMarkers: true}, - } - for _, command := range commands { - command.Run(ctx) - } + ctx.SketchSourceMerged = filterSketchSource(ctx.Sketch, strings.NewReader(ctx.SketchSourceMerged), true) err = utils.CopyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions) if err != nil { diff --git a/legacy/builder/ctags/ctags_has_issues.go b/legacy/builder/ctags/ctags_has_issues.go index 7cc693f2af6..a6cda249865 100644 --- a/legacy/builder/ctags/ctags_has_issues.go +++ b/legacy/builder/ctags/ctags_has_issues.go @@ -23,14 +23,12 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/types" ) -func (p *CTagsParser) FixCLinkageTagsDeclarations(tags []*types.CTag) { - - linesMap := p.FindCLinkageLines(tags) - for i := range tags { - - if sliceContainsInt(linesMap[tags[i].Filename], tags[i].Line) && - !strings.Contains(tags[i].PrototypeModifiers, EXTERN) { - tags[i].PrototypeModifiers = tags[i].PrototypeModifiers + " " + EXTERN +func (p *CTagsParser) FixCLinkageTagsDeclarations() { + linesMap := p.FindCLinkageLines(p.tags) + for i := range p.tags { + if sliceContainsInt(linesMap[p.tags[i].Filename], p.tags[i].Line) && + !strings.Contains(p.tags[i].PrototypeModifiers, EXTERN) { + p.tags[i].PrototypeModifiers = p.tags[i].PrototypeModifiers + " " + EXTERN } } } diff --git a/legacy/builder/ctags/ctags_parser.go b/legacy/builder/ctags/ctags_parser.go index 4ced464ae1e..a2c49851338 100644 --- a/legacy/builder/ctags/ctags_parser.go +++ b/legacy/builder/ctags/ctags_parser.go @@ -43,8 +43,8 @@ type CTagsParser struct { mainFile *paths.Path } -func (p *CTagsParser) Parse(ctagsOutput string, mainFile *paths.Path) []*types.CTag { - rows := strings.Split(ctagsOutput, "\n") +func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*types.CTag { + rows := strings.Split(string(ctagsOutput), "\n") rows = removeEmpty(rows) p.mainFile = mainFile diff --git a/legacy/builder/ctags/ctags_parser_test.go b/legacy/builder/ctags/ctags_parser_test.go index d3bb03b23a0..49711c27c18 100644 --- a/legacy/builder/ctags/ctags_parser_test.go +++ b/legacy/builder/ctags/ctags_parser_test.go @@ -30,7 +30,7 @@ func produceTags(t *testing.T, filename string) []*types.CTag { require.NoError(t, err) parser := CTagsParser{} - return parser.Parse(string(bytes), nil) + return parser.Parse(bytes, nil) } func TestCTagsParserShouldListPrototypes(t *testing.T) { diff --git a/legacy/builder/ctags/ctags_to_prototypes_test.go b/legacy/builder/ctags/ctags_to_prototypes_test.go index e73a1312069..7dcea2a0ab1 100644 --- a/legacy/builder/ctags/ctags_to_prototypes_test.go +++ b/legacy/builder/ctags/ctags_to_prototypes_test.go @@ -30,7 +30,7 @@ func producePrototypes(t *testing.T, filename string, mainFile string) ([]*types require.NoError(t, err) parser := &CTagsParser{} - parser.Parse(string(bytes), paths.New(mainFile)) + parser.Parse(bytes, paths.New(mainFile)) return parser.GeneratePrototypes() } diff --git a/legacy/builder/ctags_runner.go b/legacy/builder/ctags_runner.go index 2e42bdca234..40ef36434d1 100644 --- a/legacy/builder/ctags_runner.go +++ b/legacy/builder/ctags_runner.go @@ -26,10 +26,26 @@ import ( "github.com/pkg/errors" ) -type CTagsRunner struct{} +type CTagsRunner struct { + Source *string + TargetFileName string -func (s *CTagsRunner) Run(ctx *types.Context) error { - ctagsTargetFilePath := ctx.CTagsTargetFile + // Needed for unit-testing + CtagsOutput []byte +} + +func (r *CTagsRunner) Run(ctx *types.Context) error { + source := *r.Source + + preprocPath := ctx.PreprocPath + if err := preprocPath.MkdirAll(); err != nil { + return errors.WithStack(err) + } + + ctagsTargetFilePath := preprocPath.Join(r.TargetFileName) + if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil { + return errors.WithStack(err) + } buildProperties := properties.NewMap() buildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}") @@ -52,17 +68,14 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { command := exec.Command(parts[0], parts[1:]...) command.Env = append(os.Environ(), ctx.PackageManager.GetEnvVarsForSpawnedProcess()...) - sourceBytes, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.ShowIfVerbose /* stderr */) + ctagsOutput, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.ShowIfVerbose /* stderr */) if err != nil { return errors.WithStack(err) } - ctx.CTagsOutput = string(sourceBytes) - parser := &ctags.CTagsParser{} - - ctx.CTagsOfPreprocessedSource = parser.Parse(ctx.CTagsOutput, ctx.Sketch.MainFile) - parser.FixCLinkageTagsDeclarations(ctx.CTagsOfPreprocessedSource) + parser.Parse(ctagsOutput, ctx.Sketch.MainFile) + parser.FixCLinkageTagsDeclarations() protos, line := parser.GeneratePrototypes() if line != -1 { @@ -70,5 +83,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { } ctx.Prototypes = protos + // Needed for unit-testing + r.CtagsOutput = ctagsOutput return nil } diff --git a/legacy/builder/ctags_target_file_saver.go b/legacy/builder/ctags_target_file_saver.go index a4014662c41..261bc7c759a 100644 --- a/legacy/builder/ctags_target_file_saver.go +++ b/legacy/builder/ctags_target_file_saver.go @@ -14,30 +14,3 @@ // To purchase a commercial license, send an email to license@arduino.cc. package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" -) - -type CTagsTargetFileSaver struct { - Source *string - TargetFileName string -} - -func (s *CTagsTargetFileSaver) Run(ctx *types.Context) error { - source := *s.Source - - preprocPath := ctx.PreprocPath - if err := preprocPath.MkdirAll(); err != nil { - return errors.WithStack(err) - } - - ctagsTargetFilePath := preprocPath.Join(s.TargetFileName) - if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil { - return errors.WithStack(err) - } - - ctx.CTagsTargetFile = ctagsTargetFilePath - return nil -} diff --git a/legacy/builder/filter_sketch_source.go b/legacy/builder/filter_sketch_source.go deleted file mode 100644 index 507615a55b2..00000000000 --- a/legacy/builder/filter_sketch_source.go +++ /dev/null @@ -1,90 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "bufio" - "strconv" - "strings" - - "github.com/arduino/go-paths-helper" - - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" -) - -type FilterSketchSource struct { - Source *string - RemoveLineMarkers bool -} - -func (s *FilterSketchSource) Run(ctx *types.Context) error { - fileNames := paths.NewPathList() - fileNames.Add(ctx.Sketch.MainFile) - fileNames.AddAll(ctx.Sketch.OtherSketchFiles) - - inSketch := false - filtered := "" - - scanner := bufio.NewScanner(strings.NewReader(*s.Source)) - for scanner.Scan() { - line := scanner.Text() - if filename := parseLineMarker(line); filename != nil { - inSketch = fileNames.Contains(filename) - if inSketch && s.RemoveLineMarkers { - continue - } - } - - if inSketch { - filtered += line + "\n" - } - } - - *s.Source = filtered - return nil -} - -// Parses the given line as a gcc line marker and returns the contained -// filename. -func parseLineMarker(line string) *paths.Path { - // A line marker contains the line number and filename and looks like: - // # 123 /path/to/file.cpp - // It can be followed by zero or more flag number that indicate the - // preprocessor state and can be ignored. - // For exact details on this format, see: - // https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415 - - split := strings.SplitN(line, " ", 3) - if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' { - return nil - } - - _, err := strconv.Atoi(split[1]) - if err != nil { - return nil - } - - // If we get here, we found a # followed by a line number, so - // assume this is a line marker and see if the rest of the line - // starts with a string containing the filename - str, rest, ok := utils.ParseCppString(split[2]) - - if ok && (rest == "" || rest[0] == ' ') { - return paths.New(str) - } - return nil -} diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index 31c19ca3bc1..12c2d8f38ca 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -35,9 +35,9 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { } sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") - GCCPreprocRunner(ctx, sourceFile, ctx.PreprocPath.Join("ctags_target_for_gcc_minus_e.cpp"), ctx.IncludeFolders) + targetFile := ctx.PreprocPath.Join("sketch_merged.cpp") + GCCPreprocRunner(ctx, sourceFile, targetFile, ctx.IncludeFolders) - targetFilePath := ctx.PreprocPath.Join("ctags_target_for_gcc_minus_e.cpp") buildProperties := properties.NewMap() buildProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") buildProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") @@ -45,7 +45,7 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { buildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") buildProperties.Merge(ctx.BuildProperties) buildProperties.Merge(buildProperties.SubTree("tools").SubTree("arduino-preprocessor")) - buildProperties.SetPath("source_file", targetFilePath) + buildProperties.SetPath("source_file", targetFile) pattern := buildProperties.Get("pattern") if pattern == "" { @@ -79,7 +79,6 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { result := utils.NormalizeUTF8(buf) //fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", output) - ctx.Source = string(result) - - return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.Source), ctx.SketchBuildPath) + ctx.SketchSourceAfterArduinoPreprocessing = string(result) + return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath) } diff --git a/legacy/builder/prototypes_adder.go b/legacy/builder/prototypes_adder.go index abc816b2763..03a2cffef3a 100644 --- a/legacy/builder/prototypes_adder.go +++ b/legacy/builder/prototypes_adder.go @@ -17,19 +17,20 @@ package builder import ( "fmt" + "strconv" + "strings" + "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" - "strconv" - "strings" ) type PrototypesAdder struct{} func (s *PrototypesAdder) Run(ctx *types.Context) error { debugOutput := ctx.DebugPreprocessor - source := ctx.Source + source := ctx.SketchSourceMerged source = strings.Replace(source, "\r\n", "\n", -1) source = strings.Replace(source, "\r", "\n", -1) @@ -61,7 +62,7 @@ func (s *PrototypesAdder) Run(ctx *types.Context) error { } fmt.Println("#END OF PREPROCESSED SOURCE") } - ctx.Source = source + ctx.SketchSourceAfterArduinoPreprocessing = source return nil } diff --git a/legacy/builder/read_file_and_store_in_context.go b/legacy/builder/read_file_and_store_in_context.go deleted file mode 100644 index cdd75896387..00000000000 --- a/legacy/builder/read_file_and_store_in_context.go +++ /dev/null @@ -1,38 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -type ReadFileAndStoreInContext struct { - FileToRead *paths.Path - Target *string -} - -func (s *ReadFileAndStoreInContext) Run(ctx *types.Context) error { - bytes, err := s.FileToRead.ReadFile() - if err != nil { - return errors.WithStack(err) - } - - *s.Target = string(bytes) - - return nil -} diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 8e088eb50c1..e84e2d402dd 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -116,7 +116,7 @@ func TestBuilderEmptySketch(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "sketch1.ino.cpp.o").ExistCheck() @@ -143,7 +143,7 @@ func TestBuilderBridge(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() @@ -173,7 +173,7 @@ func TestBuilderSketchWithConfig(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "sketch_with_config.ino.cpp.o").ExistCheck() @@ -208,7 +208,7 @@ func TestBuilderBridgeTwice(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() @@ -245,7 +245,7 @@ func TestBuilderBridgeSAM(t *testing.T) { exist, err = buildPath.Join(constants.FOLDER_CORE, "avr", "dtostrf.c.d").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() @@ -286,7 +286,7 @@ func TestBuilderBridgeRedBearLab(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index 013499a461c..12194ccf744 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -19,6 +19,7 @@ import ( "strings" "testing" + bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" @@ -31,14 +32,18 @@ func TestCTagsRunner(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, - &builder.CTagsRunner{}, + ctagsRunner, } for _, command := range commands { err := command.Run(ctx) @@ -53,8 +58,7 @@ func TestCTagsRunner(t *testing.T) { "digitalCommand " + quotedSketchLocation + " /^void digitalCommand(BridgeClient client) {$/;\" kind:function line:82 signature:(BridgeClient client) returntype:void\n" + "analogCommand " + quotedSketchLocation + " /^void analogCommand(BridgeClient client) {$/;\" kind:function line:109 signature:(BridgeClient client) returntype:void\n" + "modeCommand " + quotedSketchLocation + " /^void modeCommand(BridgeClient client) {$/;\" kind:function line:149 signature:(BridgeClient client) returntype:void\n" - - require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithClass(t *testing.T) { @@ -63,14 +67,18 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, - &builder.CTagsRunner{}, + ctagsRunner, } for _, command := range commands { err := command.Run(ctx) @@ -83,7 +91,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { "set_values\t" + quotedSketchLocation + "\t/^void Rectangle::set_values (int x, int y) {$/;\"\tkind:function\tline:8\tclass:Rectangle\tsignature:(int x, int y)\treturntype:void\n" + "setup\t" + quotedSketchLocation + "\t/^void setup() {$/;\"\tkind:function\tline:13\tsignature:()\treturntype:void\n" + "loop\t" + quotedSketchLocation + "\t/^void loop() {$/;\"\tkind:function\tline:17\tsignature:()\treturntype:void\n" - require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithTypename(t *testing.T) { @@ -92,14 +100,18 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, - &builder.CTagsRunner{}, + ctagsRunner, } for _, command := range commands { err := command.Run(ctx) @@ -111,7 +123,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { "setup\t" + quotedSketchLocation + "\t/^void setup() {$/;\"\tkind:function\tline:6\tsignature:()\treturntype:void\n" + "loop\t" + quotedSketchLocation + "\t/^void loop() {}$/;\"\tkind:function\tline:10\tsignature:()\treturntype:void\n" + "func\t" + quotedSketchLocation + "\t/^typename Foo::Bar func(){$/;\"\tkind:function\tline:12\tsignature:()\treturntype:Foo::Bar\n" - require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithNamespace(t *testing.T) { @@ -120,14 +132,18 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, - &builder.CTagsRunner{}, + ctagsRunner, } for _, command := range commands { err := command.Run(ctx) @@ -138,7 +154,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { expectedOutput := "value\t" + quotedSketchLocation + "\t/^\tint value() {$/;\"\tkind:function\tline:2\tnamespace:Test\tsignature:()\treturntype:int\n" + "setup\t" + quotedSketchLocation + "\t/^void setup() {}$/;\"\tkind:function\tline:7\tsignature:()\treturntype:void\n" + "loop\t" + quotedSketchLocation + "\t/^void loop() {}$/;\"\tkind:function\tline:8\tsignature:()\treturntype:void\n" - require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithTemplates(t *testing.T) { @@ -147,14 +163,18 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, - &builder.CTagsRunner{}, + ctagsRunner, } for _, command := range commands { err := command.Run(ctx) @@ -166,5 +186,5 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { "bVar\t" + quotedSketchLocation + "\t/^c< 8 > bVar;$/;\"\tkind:variable\tline:15\n" + "aVar\t" + quotedSketchLocation + "\t/^c< 1<<8 > aVar;$/;\"\tkind:variable\tline:16\n" + "func\t" + quotedSketchLocation + "\t/^template func( c< 1< & aParam) {$/;\"\tkind:function\tline:18\tsignature:( c< 1< & aParam)\treturntype:template\n" - require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) } diff --git a/legacy/builder/test/includes_to_include_folders_test.go b/legacy/builder/test/includes_to_include_folders_test.go index 537b58d327e..a19d0ad57f2 100644 --- a/legacy/builder/test/includes_to_include_folders_test.go +++ b/legacy/builder/test/includes_to_include_folders_test.go @@ -20,6 +20,7 @@ import ( "sort" "testing" + bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" @@ -31,9 +32,13 @@ func TestIncludesToIncludeFolders(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -51,9 +56,13 @@ func TestIncludesToIncludeFoldersSketchWithIfDef(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -70,9 +79,13 @@ func TestIncludesToIncludeFoldersIRremoteLibrary(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -92,9 +105,13 @@ func TestIncludesToIncludeFoldersANewLibrary(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -119,9 +136,13 @@ func TestIncludesToIncludeFoldersDuplicateLibs(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, paths.New("user_hardware", "my_avr_platform", "avr", "libraries", "SPI", "examples", "BarometricPressureSensor", "BarometricPressureSensor.ino"), "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -147,9 +168,13 @@ func TestIncludesToIncludeFoldersDuplicateLibsWithConflictingLibsOutsideOfPlatfo ctx = prepareBuilderTestContext(t, ctx, paths.New("user_hardware", "my_avr_platform", "avr", "libraries", "SPI", "examples", "BarometricPressureSensor", "BarometricPressureSensor.ino"), "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -175,9 +200,13 @@ func TestIncludesToIncludeFoldersDuplicateLibs2(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch_usbhost", "sketch_usbhost.ino"), "arduino:samd:arduino_zero_native") defer cleanUpBuilderTestContext(t, ctx) + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -197,9 +226,13 @@ func TestIncludesToIncludeFoldersSubfolders(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index 67989677ea0..947f5bdbf02 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" @@ -37,9 +38,13 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -48,7 +53,7 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 33 "+quotedSketchLocation+"\nvoid setup();\n#line 46 "+quotedSketchLocation+"\nvoid loop();\n#line 62 "+quotedSketchLocation+"\nvoid process(BridgeClient client);\n#line 82 "+quotedSketchLocation+"\nvoid digitalCommand(BridgeClient client);\n#line 109 "+quotedSketchLocation+"\nvoid analogCommand(BridgeClient client);\n#line 149 "+quotedSketchLocation+"\nvoid modeCommand(BridgeClient client);\n#line 33 "+quotedSketchLocation+"\n", ctx.PrototypesSection) } @@ -58,9 +63,13 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -70,7 +79,7 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("SketchWithIfDef", "SketchWithIfDef.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderBaladuino(t *testing.T) { @@ -79,9 +88,13 @@ func TestPrototypesAdderBaladuino(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -91,7 +104,7 @@ func TestPrototypesAdderBaladuino(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("Baladuino", "Baladuino.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { @@ -100,9 +113,13 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -112,7 +129,7 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("CharWithEscapedDoubleQuote", "CharWithEscapedDoubleQuote.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { @@ -121,9 +138,13 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -133,7 +154,7 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("IncludeBetweenMultilineComment", "IncludeBetweenMultilineComment.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderLineContinuations(t *testing.T) { @@ -142,9 +163,13 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -154,7 +179,7 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("LineContinuations", "LineContinuations.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderStringWithComment(t *testing.T) { @@ -163,9 +188,13 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -175,7 +204,7 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("StringWithComment", "StringWithComment.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchWithStruct(t *testing.T) { @@ -184,9 +213,13 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -196,7 +229,7 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("SketchWithStruct", "SketchWithStruct.preprocessed.txt"), ctx) - obtained := strings.Replace(ctx.Source, "\r\n", "\n", -1) + obtained := strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1) // ctags based preprocessing removes the space after "dostuff", but this is still OK // TODO: remove this exception when moving to a more powerful parser preprocessed = strings.Replace(preprocessed, "void dostuff (A_NEW_TYPE * bar);", "void dostuff(A_NEW_TYPE * bar);", 1) @@ -213,9 +246,13 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -224,11 +261,11 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 13 "+quotedSketchLocation+"\nvoid setup();\n#line 17 "+quotedSketchLocation+"\nvoid loop();\n#line 13 "+quotedSketchLocation+"\n", ctx.PrototypesSection) preprocessed := LoadAndInterpolate(t, filepath.Join("sketch_with_config", "sketch_with_config.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { @@ -240,9 +277,13 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -251,7 +292,7 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "", ctx.PrototypesSection) } @@ -263,10 +304,13 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { sketchLocation := paths.New("sketch_no_functions", "sketch_no_functions.ino") quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) - + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -275,7 +319,7 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "", ctx.PrototypesSection) } @@ -288,9 +332,13 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -299,7 +347,7 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 4 "+quotedSketchLocation+"\nvoid setup();\n#line 7 "+quotedSketchLocation+"\nvoid loop();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection) } @@ -312,9 +360,13 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -323,7 +375,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") expected := "#line 1 " + quotedSketchLocation + "\nvoid setup();\n#line 2 " + quotedSketchLocation + "\nvoid loop();\n#line 4 " + quotedSketchLocation + "\nshort unsigned int testInt();\n#line 8 " + quotedSketchLocation + "\nstatic int8_t testInline();\n#line 12 " + quotedSketchLocation + "\n__attribute__((always_inline)) uint8_t testAttribute();\n#line 1 " + quotedSketchLocation + "\n" obtained := ctx.PrototypesSection @@ -347,9 +399,13 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -358,7 +414,7 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 3 "+quotedSketchLocation+"\nvoid loop();\n#line 15 "+quotedSketchLocation+"\nint8_t adalight();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection) } @@ -376,9 +432,13 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -387,7 +447,7 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 5 "+quotedSketchLocation+"\nvoid ciao();\n#line 10 "+quotedSketchLocation+"\nvoid setup();\n#line 15 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n", ctx.PrototypesSection) } @@ -404,9 +464,13 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -415,7 +479,7 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") expected := "#line 6 " + quotedSketchLocation + "\nvoid setup();\n#line 10 " + quotedSketchLocation + "\nvoid loop();\n#line 12 " + quotedSketchLocation + "\ntypename Foo::Bar func();\n#line 6 " + quotedSketchLocation + "\n" obtained := ctx.PrototypesSection // ctags based preprocessing ignores line with typename @@ -434,9 +498,13 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -445,11 +513,11 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 5 "+quotedSketchLocation+"\nvoid elseBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n", ctx.PrototypesSection) expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.txt"), ctx) - require.Equal(t, expectedSource, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, expectedSource, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { @@ -461,9 +529,13 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -472,11 +544,11 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 2 "+quotedSketchLocation+"\nvoid ifBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 2 "+quotedSketchLocation+"\n", ctx.PrototypesSection) expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.SAM.txt"), ctx) - require.Equal(t, expectedSource, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, expectedSource, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchWithConst(t *testing.T) { @@ -488,9 +560,13 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -499,7 +575,7 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 2 "+quotedSketchLocation+"\nvoid loop();\n#line 4 "+quotedSketchLocation+"\nconst __FlashStringHelper* test();\n#line 6 "+quotedSketchLocation+"\nconst int test3();\n#line 8 "+quotedSketchLocation+"\nvolatile __FlashStringHelper* test2();\n#line 10 "+quotedSketchLocation+"\nvolatile int test4();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection) } @@ -509,9 +585,13 @@ func TestPrototypesAdderSketchWithDosEol(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -531,9 +611,13 @@ func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -542,5 +626,5 @@ func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "class Foo {\nint blooper(int x) { return x+1; }\n};\n\nFoo foo;\n\n#line 7 "+quotedSketchLocation+"\nvoid setup();") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "class Foo {\nint blooper(int x) { return x+1; }\n};\n\nFoo foo;\n\n#line 7 "+quotedSketchLocation+"\nvoid setup();") } diff --git a/legacy/builder/test/read_file_and_store_in_context_test.go b/legacy/builder/test/read_file_and_store_in_context_test.go deleted file mode 100644 index c4e299e2c60..00000000000 --- a/legacy/builder/test/read_file_and_store_in_context_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package test - -import ( - "os" - "testing" - - "github.com/arduino/arduino-cli/legacy/builder" - "github.com/arduino/arduino-cli/legacy/builder/types" - paths "github.com/arduino/go-paths-helper" - "github.com/stretchr/testify/require" -) - -func TestReadFileAndStoreInContext(t *testing.T) { - filePath, err := os.CreateTemp("", "test") - NoError(t, err) - - file := paths.New(filePath.Name()) - defer file.RemoveAll() - - file.WriteFile([]byte("test test\nciao")) - - ctx := &types.Context{} - - command := &builder.ReadFileAndStoreInContext{FileToRead: file, Target: &ctx.SourceGccMinusE} - err = command.Run(ctx) - NoError(t, err) - - require.Equal(t, "test test\nciao", ctx.SourceGccMinusE) -} diff --git a/legacy/builder/test/utils_test.go b/legacy/builder/test/utils_test.go index b6f88555283..71a70be9dd6 100644 --- a/legacy/builder/test/utils_test.go +++ b/legacy/builder/test/utils_test.go @@ -65,41 +65,3 @@ func TestQuoteCppString(t *testing.T) { require.Equal(t, expected, utils.QuoteCppString(input)) } } - -func TestParseCppString(t *testing.T) { - _, _, ok := utils.ParseCppString(`foo`) - require.Equal(t, false, ok) - - _, _, ok = utils.ParseCppString(`"foo`) - require.Equal(t, false, ok) - - str, rest, ok := utils.ParseCppString(`"foo"`) - require.Equal(t, true, ok) - require.Equal(t, `foo`, str) - require.Equal(t, ``, rest) - - str, rest, ok = utils.ParseCppString(`"foo\\bar"`) - require.Equal(t, true, ok) - require.Equal(t, `foo\bar`, str) - require.Equal(t, ``, rest) - - str, rest, ok = utils.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`) - require.Equal(t, true, ok) - require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str) - require.Equal(t, ` and "then" some`, rest) - - str, rest, ok = utils.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`) - require.Equal(t, true, ok) - require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str) - require.Equal(t, ``, rest) - - str, rest, ok = utils.ParseCppString(`"/home/ççç/"`) - require.Equal(t, true, ok) - require.Equal(t, `/home/ççç/`, str) - require.Equal(t, ``, rest) - - str, rest, ok = utils.ParseCppString(`"/home/ççç/ /$sdsdd\\"`) - require.Equal(t, true, ok) - require.Equal(t, `/home/ççç/ /$sdsdd\`, str) - require.Equal(t, ``, rest) -} diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 0048d6d0c02..895c5ef043f 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -100,12 +100,17 @@ type Context struct { CollectedSourceFiles *UniqueSourceFileQueue - Sketch *sketch.Sketch - Source string - SourceGccMinusE string - + Sketch *sketch.Sketch WarningsLevel string + // Arduino sketch (.ino) to C++ (.cpp) conversion steps: + // 1. Concatenate *.ino files into a single merged source file -> SketchSourceMerged + SketchSourceMerged string + // 2. Run a pass of C++ preprocessor to remove macro definitions and ifdef-ed code -> SketchSourceAfterCppPreprocessing + SketchSourceAfterCppPreprocessing string + // 3. Do the Arduino preprocessing of the sketch (add missing prototypes) -> SketchSourceAfterArduinoPreprocessing + SketchSourceAfterArduinoPreprocessing string + // Libraries handling LibrariesManager *librariesmanager.LibrariesManager LibrariesResolver *librariesresolver.Cpp @@ -115,9 +120,6 @@ type Context struct { UseCachedLibrariesResolution bool // C++ Parsing - CTagsOutput string - CTagsTargetFile *paths.Path - CTagsOfPreprocessedSource []*CTag LineOffset int PrototypesSection string PrototypesLineWhereToInsert int diff --git a/legacy/builder/types/types.go b/legacy/builder/types/types.go index 9640d8c0c7a..3a398aabda4 100644 --- a/legacy/builder/types/types.go +++ b/legacy/builder/types/types.go @@ -122,3 +122,9 @@ type CTag struct { type Command interface { Run(ctx *Context) error } + +type BareCommand func(ctx *Context) error + +func (cmd BareCommand) Run(ctx *Context) error { + return cmd(ctx) +} diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 036fab0dc72..522f046a018 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -26,7 +26,6 @@ import ( "path/filepath" "strings" "unicode" - "unicode/utf8" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/gohasissues" @@ -301,50 +300,6 @@ func QuoteCppPath(path *paths.Path) string { return QuoteCppString(path.String()) } -// Parse a C-preprocessor string as emitted by the preprocessor. This -// is a string contained in double quotes, with any backslashes or -// quotes escaped with a backslash. If a valid string was present at the -// start of the given line, returns the unquoted string contents, the -// remainder of the line (everything after the closing "), and true. -// Otherwise, returns the empty string, the entire line and false. -func ParseCppString(line string) (string, string, bool) { - // For details about how these strings are output by gcc, see: - // https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511 - // Note that the documentation suggests all non-printable - // characters are also escaped, but the implementation does not - // actually do this. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51259 - if len(line) < 1 || line[0] != '"' { - return "", line, false - } - - i := 1 - res := "" - for { - if i >= len(line) { - return "", line, false - } - - c, width := utf8.DecodeRuneInString(line[i:]) - - switch c { - case '\\': - // Backslash, next character is used unmodified - i += width - if i >= len(line) { - return "", line, false - } - res += string(line[i]) - case '"': - // Quote, end of string - return res, line[i+width:], true - default: - res += string(c) - } - - i += width - } -} - // Normalizes an UTF8 byte slice // TODO: use it more often troughout all the project (maybe on logger interface?) func NormalizeUTF8(buf []byte) []byte {