Adding OpenTelemetry tracing

README.md

@@ -34,6 +34,13 @@ Run this simple command to download a test file
     $ ./stashcp /osgconnect/public/dweitzel/blast/queries/query1 ./
 
 
+Telementry
+----------
+
+By default, the OSDF client will collect usage information, telementry, from each invocation.  The telementry is uploaded to the OSG where it will be used to identify issues with the OSDF infrastructure and study how users are using the client.
+
+You may disable uploading telementry by setting the environment variable: `OSDF_DISABLE_TELEMENTRY`.
+
 Configuration
 -------------
 `stashcp` is affected by the environment variables:
@@ -44,6 +51,7 @@ Configuration
 | `STASHCP_MINIMUM_DOWNLOAD_SPEED`  | The lower limit a download will be cancelled, in bytes per second                                                                                                                                                                                                   |
 | `STASH_NAMESPACE_URL`             | The URL to download the namespace and cache information.  Default: https://topology.opensciencegrid.org/stashcache/namespaces                                                                                                                                                                                                 |
 
+| `OSDF_DISABLE_TELEMENTRY`         | Disable the uploading of telementry to the OSG |
 
 
 

classads/classads.go

@@ -168,7 +168,7 @@ func attributeSplitFunc(data []byte, atEOF bool) (advance int, token []byte, err
 	for i, curChar := range data {
 		if curChar == '"' && !(i > 0 && data[i-1] == '\\') {
 			insideQuotes = !insideQuotes
-		} else if curChar == ';' && !insideQuotes {
+		} else if (curChar == ';' || curChar == '\n') && !insideQuotes {
 			// Do not return the semi-colon
 			// Trim any spaces
 			return i + 1, bytes.TrimSpace(data[0:i]), nil

classads/classads_test.go

@@ -136,3 +136,21 @@ func TestAttributeSplitFunc(t *testing.T) {
 	assert.Equal(t, `Url = "stash:///osgconnect/public/$USER/file1; stash:///osgconnect/public/$USER/file2"`, attributes[1])
 
 }
+
+func TestJobAd(t *testing.T) {
+	input := `AccountingGroup = "group_opportunistic.OSG-Staff.dweitzel"
+	AcctGroup = "OSG-Staff"
+	AcctGroupUser = "dweitzel"
+	AllowedExecuteDuration = 72000
+	Arguments = ""
+	AutoClusterAttrs = "DESIRED_Sites,DiskUsage,DynamicSlot,FirstUpdateUptimeGPUsSeconds,HAS_CVMFS_connect_opensciencegrid_org,HAS_CVMFS_oasis_opensciencegrid_org,Has_MPI,HasJava,ITB_Factory,ITB_Sites,JobDurationCategory,JobUniverse,LastHeardFrom,LastUpdateUptimeGPUsSeconds,MachineLastMatchTime,MemoryUsage,OSG_NODE_VALIDATED,PartitionableSlot,ProjectName,Rank,RecentJobDurationAvg,RecentJobDurationCount,RemoteOwner,RequestCpus,RequestDisk,RequestGPUs,RequestK8SNamespace,RequestMemory,SINGULARITY_DISK_IS_FULL,SingularityImage,Slot1_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,StartOfJobUptimeGPUsSeconds,STASHCP_VERIFIED,TotalJobRuntime,undeined,UNDESIRED_Sites,UptimeGPUsSeconds,Want_MPI,XENON_DESIRED_Sites,ConcurrencyLimits,FlockTo,Requirements,IDTOKEN,FromJupyterLab,Owner,Memory,HasExcessiveLoad,IsBlackHole,HAS_MODULES,SINGULARITY_CAN_USE_SIF,FileSystemDomain"
+	AutoClusterId = 1772
+	ClusterId = 35063371
+	Cmd = "condor_exec.exe"`
+	ad, err := ReadClassAd(strings.NewReader(input))
+	assert.NoError(t, err, "ParseClassAd() failed")
+	assert.Equal(t, 1, len(ad), "ParseClassAd() returned %d ads, expected 1", len(ad))
+	accountGroup, err := ad[0].Get("AccountingGroup")
+	assert.NoError(t, err, "Get(AccountingGroup) failed")
+	assert.Equal(t, "group_opportunistic.OSG-Staff.dweitzel", accountGroup.(string))
+}

cmd/config_mgr/main.go

@@ -1,14 +1,16 @@
 package main
 
 import (
+	"context"
 	"fmt"
-	"github.com/spf13/cobra"
 	"net/url"
 	"os"
 	"path"
 
-	config "github.com/htcondor/osdf-client/v6/config"
+	"github.com/spf13/cobra"
+
 	stashcp "github.com/htcondor/osdf-client/v6"
+	config "github.com/htcondor/osdf-client/v6/config"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 )
@@ -125,7 +127,7 @@ func addTokenSubcommands(tokenCmd *cobra.Command) {
 			}
 			dest := url.URL{Path: path.Clean("/" + args[1])}
 
-			namespace, err := stashcp.MatchNamespace(args[1])
+			namespace, err := stashcp.MatchNamespace(context.Background(), args[1])
 			if err != nil {
 				fmt.Fprintln(os.Stderr, "Failed to get namespace for path:", err)
 				os.Exit(1)
@@ -284,9 +286,9 @@ func main() {
 
 	// Define the token commands
 	tokenCmd := &cobra.Command{
-		Use: "token",
+		Use:   "token",
 		Short: "Manage the available tokens",
-		Long: "Manage the available tokens",
+		Long:  "Manage the available tokens",
 	}
 	addTokenSubcommands(tokenCmd)
 

cmd/stashcp/main.go

@@ -64,6 +64,9 @@ type Options struct {
 	// Progress bars
 	ProgessBars bool `long:"progress" short:"p" description:"Show progress bars, turned on if run from a terminal"`
 
+	// Do not show progress bars
+	NoProgressBars bool `long:"no-progress" description:"Do not show progress bars, default: turned on if run from a terminal"`
+
 	// PluginInterface specifies how the output should be formatted
 	PluginInterface bool `long:"plugininterface" description:"Output in HTCondor plugin format.  Turned on if executable is named stash_plugin"`
 
@@ -78,7 +81,7 @@ var parser = flags.NewParser(&options, flags.Default)
 func main() {
 
 	stashcp.Options.Version = version
-	
+
 	// Capture the start time of the transfer
 	if _, err := parser.Parse(); err != nil {
 		if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
@@ -123,6 +126,11 @@ func main() {
 		stashcp.Options.ProgressBars = false
 	}
 
+	// Finally, if no progress bars is selected, that overrides everything
+	if options.NoProgressBars {
+		stashcp.Options.ProgressBars = false
+	}
+
 	if options.PrintNamespaces {
 		namespaces, err := stashcp.GetNamespaces()
 		if err != nil {

go.mod

@@ -7,15 +7,23 @@ require (
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/jsipprell/keyctl v1.0.4-0.20211208153515-36ca02672b6c
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.6.1
-	github.com/stretchr/testify v1.2.2
+	github.com/stretchr/testify v1.8.1
 	github.com/studio-b12/gowebdav v0.0.0-20210203212356-8244b5a5f51a
 	github.com/vbauerster/mpb/v7 v7.1.5
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/net v0.2.0
+	go.opentelemetry.io/otel v1.12.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.12.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.12.0
+	go.opentelemetry.io/otel/sdk v1.12.0
+	go.opentelemetry.io/otel/trace v1.12.0
+	golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2
+	golang.org/x/net v0.4.0
 	golang.org/x/oauth2 v0.2.0
-	golang.org/x/term v0.2.0
+	golang.org/x/term v0.3.0
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1
 )