feat(sdk): remove hex encoding for segment hash (#1805)

### Proposed Changes

*

### Checklist

- [ ] I have added or updated unit tests
- [ ] I have added or updated integration tests (if appropriate)
- [ ] I have added or updated documentation

### Testing Instructions

---------

Co-authored-by: David Mihalcik <dmihalcik@virtru.com>

Author: sujankota

sdk/manifest.go

@@ -28,6 +28,7 @@ type KeyAccess struct {
 	EncryptedMetadata string      `json:"encryptedMetadata,omitempty"`
 	KID               string      `json:"kid,omitempty"`
 	SplitID           string      `json:"sid,omitempty"`
+	SchemaVersion     string      `json:"schemaVersion,omitempty"`
 }
 
 type PolicyBinding struct {
@@ -62,6 +63,7 @@ type Manifest struct {
 	EncryptionInformation `json:"encryptionInformation"`
 	Payload               `json:"payload"`
 	Assertions            []Assertion `json:"assertions,omitempty"`
+	TDFVersion            string      `json:"schemaVersion,omitempty"`
 }
 
 type attributeObject struct {

sdk/tdf.go

@@ -23,6 +23,7 @@ import (
 )
 
 const (
+	keyAccessSchemaVersion  = "1.0"
 	maxFileSizeSupported    = 68719476736 // 64gb
 	defaultMimeType         = "application/octet-stream"
 	tdfAsZip                = "zip"
@@ -234,7 +235,8 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R
 			return nil, fmt.Errorf("io.writer.Write failed: %w", err)
 		}
 
-		segmentSig, err := calculateSignature(cipherData, tdfObject.payloadKey[:], tdfConfig.segmentIntegrityAlgorithm)
+		segmentSig, err := calculateSignature(cipherData, tdfObject.payloadKey[:],
+			tdfConfig.segmentIntegrityAlgorithm, false)
 		if err != nil {
 			return nil, fmt.Errorf("splitKey.GetSignaturefailed: %w", err)
 		}
@@ -252,7 +254,8 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R
 		readPos += readSize
 	}
 
-	rootSignature, err := calculateSignature([]byte(aggregateHash), tdfObject.payloadKey[:], tdfConfig.integrityAlgorithm)
+	rootSignature, err := calculateSignature([]byte(aggregateHash), tdfObject.payloadKey[:],
+		tdfConfig.integrityAlgorithm, false)
 	if err != nil {
 		return nil, fmt.Errorf("splitKey.GetSignaturefailed: %w", err)
 	}
@@ -299,11 +302,17 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R
 		tmpAssertion.Statement = assertion.Statement
 		tmpAssertion.AppliesToState = assertion.AppliesToState
 
-		hashOfAssertion, err := tmpAssertion.GetHash()
+		hashOfAssertionAsHex, err := tmpAssertion.GetHash()
 		if err != nil {
 			return nil, err
 		}
 
+		hashOfAssertion := make([]byte, hex.DecodedLen(len(hashOfAssertionAsHex)))
+		_, err = hex.Decode(hashOfAssertion, hashOfAssertionAsHex)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding hex string: %w", err)
+		}
+
 		var completeHashBuilder strings.Builder
 		completeHashBuilder.WriteString(aggregateHash)
 		completeHashBuilder.Write(hashOfAssertion)
@@ -320,7 +329,7 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R
 			assertionSigningKey = assertion.SigningKey
 		}
 
-		if err := tmpAssertion.Sign(string(hashOfAssertion), string(encoded), assertionSigningKey); err != nil {
+		if err := tmpAssertion.Sign(string(hashOfAssertionAsHex), string(encoded), assertionSigningKey); err != nil {
 			return nil, fmt.Errorf("failed to sign assertion: %w", err)
 		}
 
@@ -358,6 +367,8 @@ func (r *Reader) Manifest() Manifest {
 // prepare the manifest for TDF
 func (s SDK) prepareManifest(ctx context.Context, t *TDFObject, tdfConfig TDFConfig) error { //nolint:funlen,gocognit // Better readability keeping it as is
 	manifest := Manifest{}
+
+	manifest.TDFVersion = TDFSpecVersion
 	if len(tdfConfig.splitPlan) == 0 && len(tdfConfig.kasInfoList) == 0 {
 		return fmt.Errorf("%w: no key access template specified or inferred", errInvalidKasInfo)
 	}
@@ -488,6 +499,7 @@ func (s SDK) prepareManifest(ctx context.Context, t *TDFObject, tdfConfig TDFCon
 				EncryptedMetadata: encryptedMetadata,
 				SplitID:           splitID,
 				WrappedKey:        string(ocrypto.Base64Encode(wrappedKey)),
+				SchemaVersion:     keyAccessSchemaVersion,
 			}
 
 			manifest.EncryptionInformation.KeyAccessObjs = append(manifest.EncryptionInformation.KeyAccessObjs, keyAccess)
@@ -603,6 +615,8 @@ func (r *Reader) WriteTo(writer io.Writer) (int64, error) {
 		}
 	}
 
+	isLegacyTDF := r.manifest.TDFVersion == ""
+
 	var totalBytes int64
 	var payloadReadOffset int64
 	for _, seg := range r.manifest.EncryptionInformation.IntegrityInformation.Segments {
@@ -621,7 +635,7 @@ func (r *Reader) WriteTo(writer io.Writer) (int64, error) {
 			sigAlg = GMAC
 		}
 
-		payloadSig, err := calculateSignature(readBuf, r.payloadKey, sigAlg)
+		payloadSig, err := calculateSignature(readBuf, r.payloadKey, sigAlg, isLegacyTDF)
 		if err != nil {
 			return totalBytes, fmt.Errorf("splitKey.GetSignaturefailed: %w", err)
 		}
@@ -682,6 +696,7 @@ func (r *Reader) ReadAt(buf []byte, offset int64) (int, error) { //nolint:funlen
 		return 0, ErrTDFPayloadReadFail
 	}
 
+	isLegacyTDF := r.manifest.TDFVersion == ""
 	var decryptedBuf bytes.Buffer
 	var payloadReadOffset int64
 	for index, seg := range r.manifest.EncryptionInformation.IntegrityInformation.Segments {
@@ -705,7 +720,7 @@ func (r *Reader) ReadAt(buf []byte, offset int64) (int, error) { //nolint:funlen
 			sigAlg = GMAC
 		}
 
-		payloadSig, err := calculateSignature(readBuf, r.payloadKey, sigAlg)
+		payloadSig, err := calculateSignature(readBuf, r.payloadKey, sigAlg, isLegacyTDF)
 		if err != nil {
 			return 0, fmt.Errorf("splitKey.GetSignaturefailed: %w", err)
 		}
@@ -1019,18 +1034,29 @@ func (r *Reader) buildKey(_ context.Context, results []kaoResult) error {
 		}
 
 		// Get the hash of the assertion
-		hashOfAssertion, err := assertion.GetHash()
+		hashOfAssertionAsHex, err := assertion.GetHash()
 		if err != nil {
 			return fmt.Errorf("%w: failed to get hash of assertion: %w", ErrAssertionFailure{ID: assertion.ID}, err)
 		}
 
+		hashOfAssertion := make([]byte, hex.DecodedLen(len(hashOfAssertionAsHex)))
+		_, err = hex.Decode(hashOfAssertion, hashOfAssertionAsHex)
+		if err != nil {
+			return fmt.Errorf("error decoding hex string: %w", err)
+		}
+
+		isLegacyTDF := r.manifest.TDFVersion == ""
+		if isLegacyTDF {
+			hashOfAssertion = hashOfAssertionAsHex
+		}
+
 		var completeHashBuilder bytes.Buffer
 		completeHashBuilder.Write(aggregateHash.Bytes())
 		completeHashBuilder.Write(hashOfAssertion)
 
 		base64Hash := ocrypto.Base64Encode(completeHashBuilder.Bytes())
 
-		if string(hashOfAssertion) != assertionHash {
+		if string(hashOfAssertionAsHex) != assertionHash {
 			return fmt.Errorf("%w: assertion hash missmatch", ErrAssertionFailure{ID: assertion.ID})
 		}
 
@@ -1092,29 +1118,36 @@ func (r *Reader) doPayloadKeyUnwrap(ctx context.Context) error { //nolint:gocogn
 }
 
 // calculateSignature calculate signature of data of the given algorithm.
-func calculateSignature(data []byte, secret []byte, alg IntegrityAlgorithm) (string, error) {
+func calculateSignature(data []byte, secret []byte, alg IntegrityAlgorithm, isLegacyTDF bool) (string, error) {
 	if alg == HS256 {
 		hmac := ocrypto.CalculateSHA256Hmac(secret, data)
-		return hex.EncodeToString(hmac), nil
+		if isLegacyTDF {
+			return hex.EncodeToString(hmac), nil
+		}
+		return string(hmac), nil
 	}
 	if kGMACPayloadLength > len(data) {
 		return "", fmt.Errorf("fail to create gmac signature")
 	}
 
-	return hex.EncodeToString(data[len(data)-kGMACPayloadLength:]), nil
+	if isLegacyTDF {
+		return hex.EncodeToString(data[len(data)-kGMACPayloadLength:]), nil
+	}
+	return string(data[len(data)-kGMACPayloadLength:]), nil
 }
 
 // validate the root signature
 func validateRootSignature(manifest Manifest, aggregateHash, secret []byte) (bool, error) {
 	rootSigAlg := manifest.EncryptionInformation.IntegrityInformation.RootSignature.Algorithm
 	rootSigValue := manifest.EncryptionInformation.IntegrityInformation.RootSignature.Signature
+	isLegacyTDF := manifest.TDFVersion == ""
 
 	sigAlg := HS256
 	if strings.EqualFold(gmacIntegrityAlgorithm, rootSigAlg) {
 		sigAlg = GMAC
 	}
 
-	sig, err := calculateSignature(aggregateHash, secret, sigAlg)
+	sig, err := calculateSignature(aggregateHash, secret, sigAlg, isLegacyTDF)
 	if err != nil {
 		return false, fmt.Errorf("splitkey.getSignature failed:%w", err)
 	}

sdk/tdf_test.go

@@ -265,7 +265,7 @@ func (s *TDFSuite) Test_SimpleTDF() {
 		"https://example.com/attr/Classification/value/X",
 	}
 
-	expectedTdfSize := int64(2095)
+	expectedTdfSize := int64(2058)
 	tdfFilename := "secure-text.tdf"
 	plainText := "Virtru"
 	{
@@ -297,7 +297,7 @@ func (s *TDFSuite) Test_SimpleTDF() {
 		s.InDelta(float64(expectedTdfSize), float64(tdfObj.size), 32.0)
 	}
 
-	// test meta data
+	// test meta data and build meta data
 	{
 		readSeeker, err := os.Open(tdfFilename)
 		s.Require().NoError(err)
@@ -395,7 +395,7 @@ func (s *TDFSuite) Test_TDFWithAssertion() {
 			},
 			assertionVerificationKeys:    nil,
 			disableAssertionVerification: false,
-			expectedSize:                 2896,
+			expectedSize:                 2689,
 		},
 		{
 			assertions: []AssertionConfig{
@@ -428,7 +428,7 @@ func (s *TDFSuite) Test_TDFWithAssertion() {
 				DefaultKey: defaultKey,
 			},
 			disableAssertionVerification: false,
-			expectedSize:                 2896,
+			expectedSize:                 2689,
 		},
 		{
 			assertions: []AssertionConfig{
@@ -477,7 +477,7 @@ func (s *TDFSuite) Test_TDFWithAssertion() {
 				},
 			},
 			disableAssertionVerification: false,
-			expectedSize:                 3195,
+			expectedSize:                 2988,
 		},
 		{
 			assertions: []AssertionConfig{
@@ -517,7 +517,7 @@ func (s *TDFSuite) Test_TDFWithAssertion() {
 				},
 			},
 			disableAssertionVerification: false,
-			expectedSize:                 2896,
+			expectedSize:                 2689,
 		},
 		{
 			assertions: []AssertionConfig{
@@ -534,7 +534,7 @@ func (s *TDFSuite) Test_TDFWithAssertion() {
 				},
 			},
 			disableAssertionVerification: true,
-			expectedSize:                 2302,
+			expectedSize:                 2180,
 		},
 	} {
 		expectedTdfSize := test.expectedSize
@@ -643,7 +643,7 @@ func (s *TDFSuite) Test_TDFWithAssertionNegativeTests() {
 					SigningKey: defaultKey,
 				},
 			},
-			expectedSize: 2896,
+			expectedSize: 2689,
 		},
 		{
 			assertions: []AssertionConfig{
@@ -691,7 +691,7 @@ func (s *TDFSuite) Test_TDFWithAssertionNegativeTests() {
 					},
 				},
 			},
-			expectedSize: 3195,
+			expectedSize: 2988,
 		},
 		{
 			assertions: []AssertionConfig{
@@ -725,7 +725,7 @@ func (s *TDFSuite) Test_TDFWithAssertionNegativeTests() {
 			assertionVerificationKeys: &AssertionVerificationKeys{
 				DefaultKey: defaultKey,
 			},
-			expectedSize: 2896,
+			expectedSize: 2689,
 		},
 	} {
 		expectedTdfSize := test.expectedSize
@@ -940,26 +940,26 @@ func (s *TDFSuite) Test_TDF() {
 		{
 			n:           "small",
 			fileSize:    5,
-			tdfFileSize: 1557,
+			tdfFileSize: 1560,
 			checksum:    "ed968e840d10d2d313a870bc131a4e2c311d7ad09bdf32b3418147221f51a6e2",
 		},
 		{
 			n:           "small-with-mime-type",
 			fileSize:    5,
-			tdfFileSize: 1557,
+			tdfFileSize: 1560,
 			checksum:    "ed968e840d10d2d313a870bc131a4e2c311d7ad09bdf32b3418147221f51a6e2",
 			mimeType:    "text/plain",
 		},
 		{
 			n:           "1-kiB",
 			fileSize:    oneKB,
-			tdfFileSize: 2581,
+			tdfFileSize: 2598,
 			checksum:    "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a",
 		},
 		{
 			n:           "medium",
 			fileSize:    hundredMB,
-			tdfFileSize: 104866410,
+			tdfFileSize: 104866427,
 			checksum:    "cee41e98d0a6ad65cc0ec77a2ba50bf26d64dc9007f7f1c7d7df68b8b71291a6",
 		},
 	} {
@@ -1041,7 +1041,7 @@ func (s *TDFSuite) Test_KeySplits() {
 		{
 			n:           "shared",
 			fileSize:    5,
-			tdfFileSize: 2664,
+			tdfFileSize: 2759,
 			checksum:    "ed968e840d10d2d313a870bc131a4e2c311d7ad09bdf32b3418147221f51a6e2",
 			splitPlan: []keySplitStep{
 				{KAS: "https://a.kas/", SplitID: "a"},
@@ -1052,7 +1052,7 @@ func (s *TDFSuite) Test_KeySplits() {
 		{
 			n:           "split",
 			fileSize:    5,
-			tdfFileSize: 2664,
+			tdfFileSize: 2759,
 			checksum:    "ed968e840d10d2d313a870bc131a4e2c311d7ad09bdf32b3418147221f51a6e2",
 			splitPlan: []keySplitStep{
 				{KAS: "https://a.kas/", SplitID: "a"},
@@ -1063,7 +1063,7 @@ func (s *TDFSuite) Test_KeySplits() {
 		{
 			n:           "mixture",
 			fileSize:    5,
-			tdfFileSize: 3211,
+			tdfFileSize: 3351,
 			checksum:    "ed968e840d10d2d313a870bc131a4e2c311d7ad09bdf32b3418147221f51a6e2",
 			splitPlan: []keySplitStep{
 				{KAS: "https://a.kas/", SplitID: "a"},

sdk/version.go

@@ -1,6 +1,11 @@
 package sdk
 
 const (
-	Version        = "0.3.26" // SDK version // x-release-please-version
-	TDFSpecVersion = "4.2.2"  // Vesion of TDF Spec currently targeted by the SDK
+	// The latest version of TDF Spec currently targeted by the SDK.
+	// By default, new files will conform to this version of the spec
+	// and, where possible, older versions will still be readable.
+	TDFSpecVersion = "4.3.0"
+
+	// The three-part semantic version number of this SDK
+	Version = "0.3.26" // x-release-please-version
 )