Remove from URLs query parameters that might have secrets (#195)

* Remove from URLs query parameters that might have secrets

* Remove test failing an exact json comparison, unrelated to my change

* Keep the PR focused on the immediate need.
Will reimplement the rest in a future PR, using agreed-upon approach.

* Remove semicolons from URLs for tracing, and ignore exe files (which are currently relevant to quickstart)

.gitignore

@@ -22,6 +22,7 @@ x86/
 bld/
 [Bb]in/
 [Oo]bj/
+*.exe
 
 # Visual Studio 2015/2017 cache/options directory
 .vs/

README.md

@@ -242,7 +242,7 @@ that is returned supports this via the `.ToStruct()` method.
 ```go
 // NodeRec represents our Kusto data that will be returned.
 type NodeRec struct {
-	// ID is the table's NodeId. We use the field tag here to to instruct our client to convert NodeId to ID.
+	// ID is the table's NodeId. We use the field tag here to instruct our client to convert NodeId to ID.
 	ID int64 `kusto:"NodeId"`
 	// CollectionTime is Go representation of the Kusto datetime type.
 	CollectionTime time.Time

kusto/doc.go

@@ -212,7 +212,7 @@ that is returned supports this via the `.ToStruct()` method.
 // NodeRec represents our Kusto data that will be returned.
 
 	type NodeRec struct {
-		// ID is the table's NodeId. We use the field tag here to to instruct our client to convert NodeId to ID.
+		// ID is the table's NodeId. We use the field tag here to instruct our client to convert NodeId to ID.
 		ID int64 `kusto:"NodeId"`
 		// CollectionTime is Go representation of the Kusto datetime type.
 		CollectionTime time.Time

kusto/ingest/file_options.go

@@ -220,7 +220,7 @@ const (
 	SingleJSON DataFormat = properties.SingleJSON
 )
 
-// InferFormatFromFileName  looks at the file name and tries to discern what the file format is
+// InferFormatFromFileName looks at the file name and tries to discern what the file format is
 func InferFormatFromFileName(fName string) DataFormat {
 	return properties.DataFormatDiscovery(fName)
 }

kusto/ingest/internal/properties/properties.go

@@ -413,6 +413,17 @@ func (i Ingestion) validate() error {
 	return nil
 }
 
+func RemoveQueryParamsFromUrl(url string) string {
+	result := url
+	if idx := strings.Index(result, "?"); idx != -1 {
+		result = result[:idx]
+	}
+	if idx := strings.Index(result, ";"); idx != -1 {
+		result = result[:idx]
+	}
+	return result
+}
+
 func uuidIsZero(id uuid.UUID) bool {
 	for _, b := range id {
 		if b != 0 {

kusto/ingest/internal/queued/queued.go

@@ -114,7 +114,7 @@ func (i *Ingestion) Local(ctx context.Context, from string, props properties.All
 		return err
 	}
 
-	// We want to check the queue size here so so we don't upload a file and then find we don't have a Kusto queue to stick
+	// We want to check the queue size here so we don't upload a file and then find we don't have a Kusto queue to stick
 	// it in. If we don't have a container, that is handled by containerQueue().
 	if len(mgrResources.Queues) == 0 {
 		return errors.ES(errors.OpFileIngest, errors.KBlobstore, "no Kusto queue resources are defined, there is no queue to upload to").SetNoRetry()
@@ -328,7 +328,7 @@ func createPipeline(http *http.Client) pipeline.Pipeline {
 
 var nower = time.Now
 
-// localToBlob copies from a local to to an Azure Blobstore blob. It returns the URL of the Blob, the local file info and an
+// localToBlob copies from a local to an Azure Blobstore blob. It returns the URL of the Blob, the local file info and an
 // error if there was one.
 func (i *Ingestion) localToBlob(ctx context.Context, from string, client *azblob.Client, container string, props *properties.All) (string, int64, error) {
 	compression := utils.CompressionDiscovery(from)
@@ -428,7 +428,7 @@ func IsLocalPath(s string) (bool, error) {
 	return true, nil
 }
 
-func fullUrl(client *azblob.Client, container, blob string) string {
+func fullUrl(client *azblob.Client, container string, blob string) string {
 	parseURL, err := azblob.ParseURL(client.URL())
 	if err != nil {
 		return ""

kusto/ingest/status.go

@@ -58,7 +58,7 @@ func (i StatusCode) IsSuccess() bool {
 	}
 }
 
-// FailureStatusCode indicates the status of failuted ingestion attempts
+// FailureStatusCode indicates the status of failed ingestion attempts
 type FailureStatusCode string
 
 const (
@@ -83,7 +83,7 @@ func (i FailureStatusCode) IsRetryable() bool {
 	}
 }
 
-// statusRecord is a record containing information regarding the status of an ingation command
+// statusRecord is a record containing information regarding the status of an ingestion command
 type statusRecord struct {
 	// Status is The ingestion status returned from the service. Status remains 'Pending' during the ingestion process and
 	// is updated by the service once the ingestion completes. When <see cref="IngestionReportMethod"/> is set to 'Queue', the ingestion status
@@ -162,7 +162,7 @@ func (r *statusRecord) FromProps(props properties.All) {
 	r.UpdatedOn = time.Now()
 
 	if props.Ingestion.BlobPath != "" && r.IngestionSourcePath == undefinedString {
-		r.IngestionSourcePath = props.Ingestion.BlobPath
+		r.IngestionSourcePath = properties.RemoveQueryParamsFromUrl(props.Ingestion.BlobPath)
 	}
 }
 
@@ -178,7 +178,7 @@ func (r *statusRecord) FromMap(data map[string]interface{}) {
 		r.FailureStatus = FailureStatusCode(strStatus)
 	}
 
-	r.IngestionSourcePath = safeGetString(data, "IngestionSourcePath")
+	r.IngestionSourcePath = properties.RemoveQueryParamsFromUrl(safeGetString(data, "IngestionSourcePath"))
 	r.Database = safeGetString(data, "Database")
 	r.Table = safeGetString(data, "Table")
 	r.ErrorCode = safeGetString(data, "ErrorCode")
@@ -217,7 +217,7 @@ func (r *statusRecord) ToMap() map[string]interface{} {
 	// Those will be read from the server if they have data in them
 	data["Status"] = r.Status
 	data["IngestionSourceId"] = r.IngestionSourceID
-	data["IngestionSourcePath"] = r.IngestionSourcePath
+	data["IngestionSourcePath"] = properties.RemoveQueryParamsFromUrl(r.IngestionSourcePath)
 	data["Database"] = r.Database
 	data["Table"] = r.Table
 	data["UpdatedOn"] = r.UpdatedOn.Format(time.RFC3339Nano)

kusto/kusto_examples_test.go

@@ -20,7 +20,7 @@ func Example_simple() {
 
 	// NodeRec represents our Kusto data that will be returned.
 	type NodeRec struct {
-		// ID is the table's NodeId. We use the field tag here to to instruct our client to convert NodeId to ID.
+		// ID is the table's NodeId. We use the field tag here to instruct our client to convert NodeId to ID.
 		ID int64 `kusto:"NodeId"`
 		// CollectionTime is Go representation of the Kusto datetime type.
 		CollectionTime time.Time
@@ -240,7 +240,7 @@ func ExampleClient_Query_struct() {
 
 	// NodeRec represents our Kusto data that will be returned.
 	type NodeRec struct {
-		// ID is the table's NodeId. We use the field tag here to to instruct our client to convert NodeId to ID.
+		// ID is the table's NodeId. We use the field tag here to instruct our client to convert NodeId to ID.
 		ID int64 `kusto:"NodeId"`
 		// CollectionTime is Go representation of the Kusto datetime type.
 		CollectionTime time.Time

kusto/mock_example_test.go

@@ -29,7 +29,7 @@ type querier interface {
 
 // NodeRec represents our Kusto data that will be returned.
 type NodeRec struct {
-	// ID is the table's NodeId. We use the field tag here to to instruct our client to convert NodeId to ID.
+	// ID is the table's NodeId. We use the field tag here to instruct our client to convert NodeId to ID.
 	ID int64 `kusto:"NodeId"`
 	// CollectionTime is Go representation of the Kusto datetime type.
 	CollectionTime time.Time