Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(data-masking): added new ability to mask the record key and values #1291

Merged
merged 2 commits into from
Jan 2, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion application.example.yml
Original file line number Diff line number Diff line change
Expand Up @@ -295,4 +295,13 @@ akhq:
- username: header-admin
groups:
- admin


# Data masking configuration
data-masking:
filters:
- description: "Masks value for secret-key fields"
search-regex: '"(secret-key)":".*"'
replacement: '"$1":"xxxx"'
- description: "Masks last digits of phone numbers"
search-regex: '"([\+]?[(]?[0-9]{3}[)]?[-\s\.]?[0-9]{3}[-\s\.]?)[0-9]{4,6}"'
replacement: '"$1xxxx"'
15 changes: 15 additions & 0 deletions docs/docs/configuration/akhq.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,18 @@ akhq:
value: "none"
```
## Data Masking
If you want to hide some data in your records, you can configure this with the following filters.
These will be applied to all record values and keys.
```yaml
akhq:
security:
data-masking:
filters:
- description: "Masks value for secret-key fields"
search-regex: '"(secret-key)":".*"'
replacement: '"$1":"xxxx"'
- description: "Masks last digits of phone numbers"
search-regex: '"([\+]?[(]?[0-9]{3}[)]?[-\s\.]?[0-9]{3}[-\s\.]?)[0-9]{4,6}"'
replacement: '"$1xxxx"'
```
13 changes: 13 additions & 0 deletions src/main/java/org/akhq/configs/DataMasking.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package org.akhq.configs;

import io.micronaut.context.annotation.ConfigurationProperties;
import lombok.Data;

import java.util.ArrayList;
import java.util.List;

@ConfigurationProperties("akhq.security.data-masking")
@Data
public class DataMasking {
List<DataMaskingFilter> filters = new ArrayList<>();
}
12 changes: 12 additions & 0 deletions src/main/java/org/akhq/configs/DataMaskingFilter.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package org.akhq.configs;

import io.micronaut.context.annotation.EachProperty;
import lombok.Data;

@EachProperty("filters")
@Data
public class DataMaskingFilter {
String description;
String searchRegex;
String replacement;
}
4 changes: 4 additions & 0 deletions src/main/java/org/akhq/models/Record.java
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,10 @@ public void setValue(String value) {
this.value = value;
}

public void setKey(String key) {
this.key = key;
}

public void setTruncated(Boolean truncated) {
this.truncated = truncated;
}
Expand Down
12 changes: 8 additions & 4 deletions src/main/java/org/akhq/repositories/RecordRepository.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.akhq.modules.schemaregistry.RecordWithSchemaSerializerFactory;
import org.akhq.utils.AvroToJsonSerializer;
import org.akhq.utils.Debug;
import org.akhq.utils.MaskingUtils;
import org.apache.kafka.clients.admin.DeletedRecords;
import org.apache.kafka.clients.admin.RecordsToDelete;
import org.apache.kafka.clients.consumer.*;
Expand Down Expand Up @@ -72,6 +73,9 @@ public class RecordRepository extends AbstractRepository {
@Inject
private AvroWireFormatConverter avroWireFormatConverter;

@Inject
private MaskingUtils maskingUtils;

@Value("${akhq.topic-data.poll-timeout:1000}")
protected int pollTimeout;

Expand Down Expand Up @@ -443,7 +447,7 @@ private ConsumerRecords<byte[], byte[]> poll(KafkaConsumer<byte[], byte[]> consu
private Record newRecord(ConsumerRecord<byte[], byte[]> record, String clusterId, Topic topic) {
SchemaRegistryType schemaRegistryType = this.schemaRegistryRepository.getSchemaRegistryType(clusterId);
SchemaRegistryClient client = this.kafkaModule.getRegistryClient(clusterId);
return new Record(
return maskingUtils.maskRecord(new Record(
client,
record,
this.schemaRegistryRepository.getSchemaRegistryType(clusterId),
Expand All @@ -456,13 +460,13 @@ private Record newRecord(ConsumerRecord<byte[], byte[]> record, String clusterId
avroWireFormatConverter.convertValueToWireFormat(record, client,
this.schemaRegistryRepository.getSchemaRegistryType(clusterId)),
topic
);
));
}

private Record newRecord(ConsumerRecord<byte[], byte[]> record, BaseOptions options, Topic topic) {
SchemaRegistryType schemaRegistryType = this.schemaRegistryRepository.getSchemaRegistryType(options.clusterId);
SchemaRegistryClient client = this.kafkaModule.getRegistryClient(options.clusterId);
return new Record(
return maskingUtils.maskRecord(new Record(
client,
record,
schemaRegistryType,
Expand All @@ -475,7 +479,7 @@ private Record newRecord(ConsumerRecord<byte[], byte[]> record, BaseOptions opti
avroWireFormatConverter.convertValueToWireFormat(record, client,
this.schemaRegistryRepository.getSchemaRegistryType(options.clusterId)),
topic
);
));
}

public List<RecordMetadata> produce(
Expand Down
38 changes: 38 additions & 0 deletions src/main/java/org/akhq/utils/MaskingUtils.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package org.akhq.utils;

import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import org.akhq.configs.DataMasking;
import org.akhq.configs.DataMaskingFilter;
import org.akhq.models.Record;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Singleton
public class MaskingUtils {
private static final Logger LOG = LoggerFactory.getLogger(MaskingUtils.class);

@Inject
DataMasking dataMasking;

public Record maskRecord(Record record) {
LOG.trace("masking record");

String value = record.getValue();
String key = record.getKey();

for (DataMaskingFilter filter : dataMasking.getFilters()) {
if (value != null) {
value = value.replaceAll(filter.getSearchRegex(), filter.getReplacement());
}
if (key != null) {
key = key.replaceAll(filter.getSearchRegex(), filter.getReplacement());
}
}

record.setValue(value);
record.setKey(key);

return record;
}
}
64 changes: 64 additions & 0 deletions src/test/java/org/akhq/utils/MaskingUtilsTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package org.akhq.utils;

import io.micronaut.context.ApplicationContext;
import org.akhq.models.Record;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;

class MaskingUtilsTest {

@Test
void shouldMasksRecordValue() {
ApplicationContext ctx = ApplicationContext.run("data-masking");
MaskingUtils maskingUtils = ctx.getBean(MaskingUtils.class);

Record record = new Record();
record.setValue("{\"secret-key\":\"my-secret-value\"}");

Record maskedRecord = maskingUtils.maskRecord(record);

assertEquals(
"{\"secret-key\":\"xxxx\"}",
maskedRecord.getValue()
);

ctx.close();
}

@Test
void shouldMasksRecordKey() {
ApplicationContext ctx = ApplicationContext.run("data-masking");
MaskingUtils maskingUtils = ctx.getBean(MaskingUtils.class);

Record record = new Record();
record.setKey("{\"secret-key\":\"my-secret-value\"}");

Record maskedRecord = maskingUtils.maskRecord(record);

assertEquals(
"{\"secret-key\":\"xxxx\"}",
maskedRecord.getKey()
);

ctx.close();
}

@Test
void shouldReturnGroupsToAllowPartialMasking() {
ApplicationContext ctx = ApplicationContext.run("data-masking");
MaskingUtils maskingUtils = ctx.getBean(MaskingUtils.class);

Record record = new Record();
record.setValue("{\"some-key\":\"+12092503766\"}");

Record maskedRecord = maskingUtils.maskRecord(record);

assertEquals(
"{\"some-key\":\"+120925xxxx\"}",
maskedRecord.getValue()
);

ctx.close();
}
}
10 changes: 10 additions & 0 deletions src/test/resources/application-data-masking.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
akhq:
security:
data-masking:
filters:
- description: "Masks value for secret-key fields"
search-regex: '"(secret-key)":".*"'
replacement: '"$1":"xxxx"'
- description: "Masks last digits of phone numbers"
search-regex: '"([\+]?[(]?[0-9]{3}[)]?[-\s\.]?[0-9]{3}[-\s\.]?)[0-9]{4,6}"'
replacement: '"$1xxxx"'