Skip to content

Commit 54ebf2c

Browse files
himanshu219himanshu219
committed
limit raised to 1024 for all metadata fields
1 parent 424323f commit 54ebf2c

File tree

7 files changed

+24
-14
lines changed

7 files changed

+24
-14
lines changed

‎AppendBlobReader/src/consumer.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ async function appendBlobStreamMessageHandlerv2(context, serviceBusTask) {
264264
* @returns {string} - The new string, truncated if necessary.
265265
*/
266266
function checkAndTruncate(data) {
267-
const maxLength = 128;
267+
const maxLength = 1024;
268268

269269
// Check if the string length exceeds the maximum length
270270
if (data.length > maxLength) {
@@ -297,9 +297,9 @@ function setSourceCategory(serviceBusTask, options) {
297297
});
298298
options.metadata["sourceFields"] = customFieldsArr.join();
299299
}
300-
options.metadata["sourceHost"] = `${serviceBusTask.storageName}/${serviceBusTask.containerName}`
300+
options.metadata["sourceHost"] = checkAndTruncate(`${serviceBusTask.storageName}/${serviceBusTask.containerName}`);
301301
// context.log(serviceBusTask.blobName, serviceBusTask.storageName,serviceBusTask.containerName);
302-
// options.metadata["sourceCategory"] = "custom_source_category";
302+
// options.metadata["sourceCategory"] = checkAndTruncate("custom_source_category");
303303
options.metadata["sourceName"] = checkAndTruncate(serviceBusTask.blobName);
304304
}
305305

‎AppendBlobReader/target/consumer_build/AppendBlobTaskConsumer/index.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ async function appendBlobStreamMessageHandlerv2(context, serviceBusTask) {
264264
* @returns {string} - The new string, truncated if necessary.
265265
*/
266266
function checkAndTruncate(data) {
267-
const maxLength = 128;
267+
const maxLength = 1024;
268268

269269
// Check if the string length exceeds the maximum length
270270
if (data.length > maxLength) {
@@ -297,9 +297,9 @@ function setSourceCategory(serviceBusTask, options) {
297297
});
298298
options.metadata["sourceFields"] = customFieldsArr.join();
299299
}
300-
options.metadata["sourceHost"] = `${serviceBusTask.storageName}/${serviceBusTask.containerName}`
300+
options.metadata["sourceHost"] = checkAndTruncate(`${serviceBusTask.storageName}/${serviceBusTask.containerName}`);
301301
// context.log(serviceBusTask.blobName, serviceBusTask.storageName,serviceBusTask.containerName);
302-
// options.metadata["sourceCategory"] = "custom_source_category";
302+
// options.metadata["sourceCategory"] = checkAndTruncate("custom_source_category");
303303
options.metadata["sourceName"] = checkAndTruncate(serviceBusTask.blobName);
304304
}
305305

‎AppendBlobReader/tests/test_appendblobreader.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,9 @@ def test_03_func_logs(self):
183183
f"append blob file's record count: {record_unsupported_extension_count}, logs with unsupported blob extension should not be ingested")
184184

185185
# Verify with a very long append blob filename (1024 characters)
186-
if len(self.test_filename) > 128:
186+
# _sourceCategory, _sourceHost, _sourceName have this limit
187+
maxMetadataLength = 1024
188+
if len(self.test_filename) > maxMetadataLength:
187189
expected_filename = self.test_filename[:60] + "..." + self.test_filename[-60:]
188190
else:
189191
expected_filename = self.test_filename

‎BlockBlobReader/src/consumer.js

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,8 @@ function messageHandler(serviceBusTask, context, sumoClient) {
414414
* @returns {string} - The new string, truncated if necessary.
415415
*/
416416
function checkAndTruncate(data) {
417-
const maxLength = 128;
417+
418+
const maxLength = 1024;
418419

419420
// Check if the string length exceeds the maximum length
420421
if (data.length > maxLength) {
@@ -437,6 +438,7 @@ function checkAndTruncate(data) {
437438
* metadata.sourceCategory attribute sets the source category
438439
*/
439440
function setSourceCategory(serviceBusTask, options) {
441+
440442
options.metadata = options.metadata || {};
441443
// make sure to add custom fileds in HTTP source in sumologic portal: https://help.sumologic.com/docs/manage/fields/#collector-and-source-fields, otherwise these fileds will be dropped.
442444
let customFields = {}; // { "containername": serviceBusTask.containerName, "storagename": serviceBusTask.storageName };
@@ -447,7 +449,7 @@ function setSourceCategory(serviceBusTask, options) {
447449
});
448450
options.metadata["sourceFields"] = customFieldsArr.join();
449451
}
450-
options.metadata["sourceHost"] = `${serviceBusTask.storageName}/${serviceBusTask.containerName}`
452+
options.metadata["sourceHost"] = checkAndTruncate(`${serviceBusTask.storageName}/${serviceBusTask.containerName}`);
451453
// context.log(serviceBusTask.blobName, serviceBusTask.storageName,serviceBusTask.containerName);
452454
// options.metadata["sourceCategory"] = "custom_source_category";
453455
options.metadata["sourceName"] = checkAndTruncate(serviceBusTask.blobName);

‎BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,8 @@ function messageHandler(serviceBusTask, context, sumoClient) {
414414
* @returns {string} - The new string, truncated if necessary.
415415
*/
416416
function checkAndTruncate(data) {
417-
const maxLength = 128;
417+
418+
const maxLength = 1024;
418419

419420
// Check if the string length exceeds the maximum length
420421
if (data.length > maxLength) {
@@ -437,6 +438,7 @@ function checkAndTruncate(data) {
437438
* metadata.sourceCategory attribute sets the source category
438439
*/
439440
function setSourceCategory(serviceBusTask, options) {
441+
440442
options.metadata = options.metadata || {};
441443
// make sure to add custom fileds in HTTP source in sumologic portal: https://help.sumologic.com/docs/manage/fields/#collector-and-source-fields, otherwise these fileds will be dropped.
442444
let customFields = {}; // { "containername": serviceBusTask.containerName, "storagename": serviceBusTask.storageName };
@@ -447,7 +449,7 @@ function setSourceCategory(serviceBusTask, options) {
447449
});
448450
options.metadata["sourceFields"] = customFieldsArr.join();
449451
}
450-
options.metadata["sourceHost"] = `${serviceBusTask.storageName}/${serviceBusTask.containerName}`
452+
options.metadata["sourceHost"] = checkAndTruncate(`${serviceBusTask.storageName}/${serviceBusTask.containerName}`);
451453
// context.log(serviceBusTask.blobName, serviceBusTask.storageName,serviceBusTask.containerName);
452454
// options.metadata["sourceCategory"] = "custom_source_category";
453455
options.metadata["sourceName"] = checkAndTruncate(serviceBusTask.blobName);

‎BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/index.js

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,8 @@ function messageHandler(serviceBusTask, context, sumoClient) {
414414
* @returns {string} - The new string, truncated if necessary.
415415
*/
416416
function checkAndTruncate(data) {
417-
const maxLength = 128;
417+
418+
const maxLength = 1024;
418419

419420
// Check if the string length exceeds the maximum length
420421
if (data.length > maxLength) {
@@ -437,6 +438,7 @@ function checkAndTruncate(data) {
437438
* metadata.sourceCategory attribute sets the source category
438439
*/
439440
function setSourceCategory(serviceBusTask, options) {
441+
440442
options.metadata = options.metadata || {};
441443
// make sure to add custom fileds in HTTP source in sumologic portal: https://help.sumologic.com/docs/manage/fields/#collector-and-source-fields, otherwise these fileds will be dropped.
442444
let customFields = {}; // { "containername": serviceBusTask.containerName, "storagename": serviceBusTask.storageName };
@@ -447,7 +449,7 @@ function setSourceCategory(serviceBusTask, options) {
447449
});
448450
options.metadata["sourceFields"] = customFieldsArr.join();
449451
}
450-
options.metadata["sourceHost"] = `${serviceBusTask.storageName}/${serviceBusTask.containerName}`
452+
options.metadata["sourceHost"] = checkAndTruncate(`${serviceBusTask.storageName}/${serviceBusTask.containerName}`);
451453
// context.log(serviceBusTask.blobName, serviceBusTask.storageName,serviceBusTask.containerName);
452454
// options.metadata["sourceCategory"] = "custom_source_category";
453455
options.metadata["sourceName"] = checkAndTruncate(serviceBusTask.blobName);

‎BlockBlobReader/tests/test_blobreader.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,9 @@ def upload_file_of_unknown_extension(self):
9393
def get_full_testlog_file_name(self):
9494
# Verify with a very long append blob filename (1024 characters)
9595
file_ext = f".{self.log_type}"
96-
if len(self.test_filename) > 128:
96+
# _sourceCategory, _sourceHost, _sourceName have this limit
97+
maxMetadataLength = 1024
98+
if len(self.test_filename) > maxMetadataLength:
9799
expected_filename = self.test_filename[:60] + "..." + self.test_filename[-(60-len(file_ext)):] + file_ext
98100
else:
99101
expected_filename = self.test_filename

0 commit comments

Comments
 (0)