Generate semantic conventions according to specification version

RELEASING.md

@@ -4,9 +4,10 @@
 
 If a new version of the OpenTelemetry Specification has been released it will be necessary to generate a new
 semantic convention package from the YAML definitions in the specification repository. There is a utility in
-`internal/tools/semconv-gen` that can be used to generate the `semconv` package. This will ideally be done
-shortly after the specification release is tagged, but it is also good practice to ensure that current conventions
-are current before creating a release tag.
+`internal/tools/semconv-gen` that can be used to generate the a package with the name matching the specification
+version number under the `semconv` package. This will ideally be done soon after the specification release is
+tagged. Make sure that the specification repo contains a checkout of the the latest tagged release so that the
+generated files match the released semantic conventions.
 
 There are currently two categories of semantic conventions that must be generated, `resource` and `trace`.
 
@@ -17,7 +18,7 @@ go run generate.go -i /path/to/specification/repo/semantic_conventions/trace
 ```
 
 Using default values for all options other than `input` will result in using the `template.j2` template to
-generate `resource.go` and `trace.go` in `/path/to/otelgo/repo/semconv`.
+generate `resource.go` and `trace.go` in `/path/to/otelgo/repo/semconv/<version>`.
 
 ## Pre-Release
 

internal/tools/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/golangci/golangci-lint v1.40.1
 	github.com/itchyny/gojq v0.12.3
 	github.com/spf13/pflag v1.0.5
+	golang.org/x/mod v0.4.2
 	golang.org/x/tools v0.1.2-0.20210512205948-8287d5da45e4
 )
 

internal/tools/semconv-gen/generator.go

@@ -19,20 +19,27 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 
 	flag "github.com/spf13/pflag"
+	"golang.org/x/mod/semver"
 )
 
 func main() {
+	// Plain log output, no timestamps.
+	log.SetFlags(0)
+
 	cfg := config{}
-	flag.StringVarP(&cfg.inputPath, "input", "i", "", "Path to semantic convention definition YAML")
-	flag.StringVarP(&cfg.outputPath, "output", "o", "semconv", "Path to output target. Must be either an absolute path or relative to the repository root.")
+	flag.StringVarP(&cfg.inputPath, "input", "i", "", "Path to semantic convention definition YAML. Should be a directory in the specification git repository.")
+	flag.StringVarP(&cfg.specVersion, "specver", "s", "", "Version of semantic convention to generate. Must be an existing version tag in the specification git repository.")
+	flag.StringVarP(&cfg.outputPath, "output", "o", "", "Path to output target. Must be either an absolute path or relative to the repository root. If unspecified will output to a sub-directory with the name matching the version number specified via --specver flag.")
 	flag.StringVarP(&cfg.containerImage, "container", "c", "otel/semconvgen", "Container image ID")
 	flag.StringVarP(&cfg.outputFilename, "filename", "f", "", "Filename for templated output. If not specified 'basename(inputPath).go' will be used.")
 	flag.StringVarP(&cfg.templateFilename, "template", "t", "template.j2", "Template filename")
@@ -50,7 +57,7 @@ func main() {
 		panic(err)
 	}
 
-	err = fixIdentifiers(cfg.outputFilename)
+	err = fixIdentifiers(cfg)
 	if err != nil {
 		panic(err)
 	}
@@ -67,17 +74,29 @@ type config struct {
 	outputFilename   string
 	templateFilename string
 	containerImage   string
+	specVersion      string
 }
 
 func validateConfig(cfg config) (config, error) {
-	if cfg.inputPath == "" {
-		return config{}, errors.New("input path must be provided")
-	}
-
 	if cfg.outputFilename == "" {
 		cfg.outputFilename = fmt.Sprintf("%s.go", path.Base(cfg.inputPath))
 	}
 
+	if cfg.specVersion == "" {
+		// Find the latest version of the specification and use it for generation.
+		var err error
+		cfg.specVersion, err = findLatestSpecVersion(cfg)
+		if err != nil {
+			return config{}, err
+		}
+	}
+
+	if cfg.outputPath == "" {
+		// If output path is unspecified put it under a sub-directory with a name matching
+		// the version of semantic convention under the semconv directory.
+		cfg.outputPath = path.Join("semconv", cfg.specVersion)
+	}
+
 	if !path.IsAbs(cfg.outputPath) {
 		root, err := findRepoRoot()
 		if err != nil {
@@ -106,8 +125,8 @@ func render(cfg config) error {
 	}
 	defer os.RemoveAll(tmpDir)
 
-	inputPath := path.Join(tmpDir, "input")
-	err = os.Mkdir(inputPath, 0700)
+	specCheckoutPath := path.Join(tmpDir, "input")
+	err = os.Mkdir(specCheckoutPath, 0700)
 	if err != nil {
 		return fmt.Errorf("unable to create input directory: %w", err)
 	}
@@ -118,10 +137,13 @@ func render(cfg config) error {
 		return fmt.Errorf("unable to create output directory: %w", err)
 	}
 
-	err = exec.Command("cp", "-a", cfg.inputPath, inputPath).Run()
+	// Checkout the specification repo to a temp dir. This will be the input
+	// for the generator.
+	doneFunc, err := checkoutSpecToDir(cfg, specCheckoutPath)
 	if err != nil {
-		return fmt.Errorf("unable to copy input to temp directory: %w", err)
+		return err
 	}
+	defer doneFunc()
 
 	err = exec.Command("cp", cfg.templateFilename, tmpDir).Run()
 	if err != nil {
@@ -131,7 +153,7 @@ func render(cfg config) error {
 	cmd := exec.Command("docker", "run", "--rm",
 		"-v", fmt.Sprintf("%s:/data", tmpDir),
 		cfg.containerImage,
-		"--yaml-root", path.Join("/data/input", path.Base(cfg.inputPath)),
+		"--yaml-root", path.Join("/data/input/semantic_conventions/", path.Base(cfg.inputPath)),
 		"code",
 		"--template", path.Join("/data", path.Base(cfg.templateFilename)),
 		"--output", path.Join("/data/output", path.Base(cfg.outputFilename)),
@@ -141,6 +163,10 @@ func render(cfg config) error {
 		return fmt.Errorf("unable to render template: %w", err)
 	}
 
+	err = os.MkdirAll(cfg.outputPath, 0700)
+	if err != nil {
+		return fmt.Errorf("unable to create output directory %s: %w", cfg.outputPath, err)
+	}
 	err = exec.Command("cp", path.Join(tmpDir, "output", path.Base(cfg.outputFilename)), cfg.outputPath).Run()
 	if err != nil {
 		return fmt.Errorf("unable to copy result to target: %w", err)
@@ -149,6 +175,84 @@ func render(cfg config) error {
 	return nil
 }
 
+type semVerSlice []string
+
+func (s semVerSlice) Len() int {
+	return len(s)
+}
+
+func (s semVerSlice) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s semVerSlice) Less(i, j int) bool {
+	return semver.Compare(s[i], s[j]) < 0
+}
+
+// findLatestSpecVersion finds the latest specification version number and checkouts
+// that version in the repo's working directory.
+func findLatestSpecVersion(cfg config) (string, error) {
+	// List all tags in the specification repo. All released version numbers are tags
+	// in the repo.
+	cmd := exec.Command("git", "tag")
+	// The specification repo is in cfg.inputPath.
+	cmd.Dir = cfg.inputPath
+	output, err := cmd.Output()
+	if err != nil {
+		return "", fmt.Errorf("unable to exec %s: %w", cmd.String(), err)
+	}
+
+	// Split the output: each line is a tag.
+	lines := strings.Split(string(output), "\n")
+
+	// Copy valid semver version numbers to a slice.
+	var versions semVerSlice
+	for _, line := range lines {
+		ver := line
+		if semver.IsValid(ver) {
+			versions = append(versions, ver)
+		}
+	}
+
+	// Sort it according to semver rules.
+	sort.Sort(versions)
+
+	if len(versions) == 0 {
+		return "", fmt.Errorf("no version tags found in the specification repo at %s", cfg.inputPath)
+	}
+
+	// Use the latest version number.
+	lastVer := versions[len(versions)-1]
+	return lastVer, nil
+}
+
+// checkoutSpecToDir checks out the specification repository to the toDir.
+// Returned doneFunc should be called when the directory is no longer needed and can be
+// cleaned up.
+func checkoutSpecToDir(cfg config, toDir string) (doneFunc func(), err error) {
+	// Checkout the selected tag to make sure we use the correct version of semantic
+	// convention yaml files as the input. We will checkout the worktree to a temporary toDir.
+	cmd := exec.Command("git", "worktree", "add", toDir, cfg.specVersion)
+	// The specification repo is in cfg.inputPath.
+	cmd.Dir = cfg.inputPath
+	err = cmd.Run()
+	if err != nil {
+		return nil, fmt.Errorf("unable to exec %s: %w", cmd.String(), err)
+	}
+
+	doneFunc = func() {
+		// Remove the worktree when it is no longer needed.
+		cmd := exec.Command("git", "worktree", "remove", "-f", toDir)
+		cmd.Dir = cfg.inputPath
+		err := cmd.Run()
+		if err != nil {
+			log.Printf("Could not cleanup spec repo worktree, unable to exec %s: %s\n", cmd.String(), err.Error())
+		}
+	}
+
+	return doneFunc, nil
+}
+
 func findRepoRoot() (string, error) {
 	start, err := os.Getwd()
 	if err != nil {
@@ -277,8 +381,8 @@ var replacements = map[string]string{
 	"Lineno":        "LineNumber",
 }
 
-func fixIdentifiers(fn string) error {
-	data, err := ioutil.ReadFile(fn)
+func fixIdentifiers(cfg config) error {
+	data, err := ioutil.ReadFile(cfg.outputFilename)
 	if err != nil {
 		return fmt.Errorf("unable to read file: %w", err)
 	}
@@ -298,7 +402,12 @@ func fixIdentifiers(fn string) error {
 		data = bytes.ReplaceAll(data, []byte(cur), []byte(repl))
 	}
 
-	err = ioutil.WriteFile(fn, data, 0644)
+	// Inject the correct import path.
+	packageDir := path.Base(path.Dir(cfg.outputFilename))
+	importPath := fmt.Sprintf(`"go.opentelemetry.io/otel/semconv/%s"`, packageDir)
+	data = bytes.ReplaceAll(data, []byte(`[[IMPORTPATH]]`), []byte(importPath))
+
+	err = ioutil.WriteFile(cfg.outputFilename, data, 0644)
 	if err != nil {
 		return fmt.Errorf("unable to write updated file: %w", err)
 	}

internal/tools/semconv-gen/template.j2

@@ -46,7 +46,7 @@ Note: {{ attr.note }}
 
 // Code generated from semantic convention specification. DO NOT EDIT.
 
-package semconv  // import "go.opentelemetry.io/otel/semconv"
+package semconv  // import [[IMPORTPATH]]
 
 import "go.opentelemetry.io/otel/attribute"