Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
13686b1
control-service: add aws ses smtp
yonitoo Mar 21, 2024
682577a
Google Java Format
Mar 21, 2024
ac9706c
control-service: change gitlab ci to build a test image for ses
yonitoo Mar 22, 2024
0bdb710
vdk-dag: add status checks functionalities (#3250)
duyguHsnHsn Mar 21, 2024
f09b1fc
vdk-core: env var precedence over secrets config (#3239)
DeltaMichael Mar 22, 2024
105a418
vdk-core: add db_connection_after_operation (#3254)
duyguHsnHsn Mar 22, 2024
ea602ce
control-service: explicitly pass the test image tag
yonitoo Mar 22, 2024
7fd1682
Merge branch 'main' into person/yonitoo/test-aws-smtp
yonitoo Mar 22, 2024
e1a8800
control-service: add AWS profile for the AmazonEmailNotification class
yonitoo Mar 25, 2024
aceecb9
add debug logs
yonitoo Mar 26, 2024
a0ae1ec
Google Java Format
Mar 26, 2024
b6899d8
vdk-oracle: remove secrets override (#3220)
duyguHsnHsn Mar 22, 2024
71c6010
vdk-oracle: Pass ingestion payload rows in uniform batches (#3194)
DeltaMichael Mar 22, 2024
75e3bbf
vdk-smarter: migrate decorate_operation to before_operation (#3255)
DeltaMichael Mar 26, 2024
547f687
vdk-lineage: migrate decorate_operation to before_operation (#3253)
DeltaMichael Mar 26, 2024
e34d2bf
vdk-impala: adapt lineage to new db operations (#3258)
duyguHsnHsn Mar 26, 2024
dfc8e00
Merge branch 'main' into person/yonitoo/test-aws-smtp
yonitoo Mar 26, 2024
9794e72
change to prints
yonitoo Mar 26, 2024
720039e
control-service: add a few more logs
yonitoo Mar 27, 2024
295d5f0
control-service: change dest due to lack of permissions
yonitoo Mar 27, 2024
f4a9655
Google Java Format
Mar 27, 2024
a041930
control-service: amazon ses uses real values.
murphp15 Mar 27, 2024
87ca661
Google Java Format
Mar 27, 2024
d037f98
Merge branch 'main' into person/murphp15/amazon_ses_use_real_config
murphp15 Mar 27, 2024
6e5dac2
control-service: add condition to prefer amazon when ARN is present
yonitoo Mar 28, 2024
496a097
Merge branch 'main' into person/murphp15/amazon_ses_use_real_config
yonitoo Mar 28, 2024
96296de
control-service: fix prefix
yonitoo Mar 28, 2024
cfd8e97
control-service: change smtp condition
yonitoo Mar 29, 2024
6f39c2e
control-service: amazon ses uses real values.
murphp15 Mar 29, 2024
bb4a3ba
control-service: amazon ses uses real values.
murphp15 Mar 29, 2024
0997522
control-service: amazon ses uses real values.
murphp15 Mar 29, 2024
758cb10
control-service: cleanup the changes from redundant logs
yonitoo Mar 29, 2024
3ff6061
control-service: remove unused logger
yonitoo Mar 29, 2024
8e8d51b
control-service: change logs to debug
yonitoo Mar 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ dependencies { // Implementation dependencies are found on compile classpath of
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.retry:spring-retry'

implementation 'io.awspring.cloud:spring-cloud-starter-aws-ses:2.4.4'
// Hashicorp Vault client libraries
implementation versions.'org.springframework.vault:spring-vault-core'

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Copyright 2023-2024 Broadcom
* SPDX-License-Identifier: Apache-2.0
*/

package com.vmware.taurus.service.notification;

import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClientBuilder;
import com.amazonaws.services.simpleemail.model.*;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.mail.MessagingException;
import javax.mail.internet.InternetAddress;
import java.util.Arrays;

/**
* The AmazonEmailNotification class is responsible for implementing the email notifications
* functionality in AWS using SES. The single exposed send method accepts pre-build notification
* object.
*
* @see DataJobNotification
* @see NotificationContent
*/
public class AmazonSesEmailNotification implements EmailNotification {

static Logger log = LoggerFactory.getLogger(AmazonSesEmailNotification.class);
private final String roleArn;
private final String region;

public AmazonSesEmailNotification(EmailConfiguration emailConfiguration) {
this.region = emailConfiguration.smtpWithPrefix().get("mail.smtp.host");
this.roleArn = emailConfiguration.getUsername();
}

@Override
public void send(NotificationContent notificationContent) throws MessagingException {
String roleSessionName = "email-session";

STSAssumeRoleSessionCredentialsProvider credentialsProvider =
new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, roleSessionName)
.withStsClient(
AWSSecurityTokenServiceClientBuilder.standard()
.withCredentials(new DefaultAWSCredentialsProviderChain())
.withRegion(this.region)
.build())
.build();

var sesClient =
AmazonSimpleEmailServiceClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(credentialsProvider.getCredentials()))
.withRegion(this.region)
.build();
log.debug("Send AmazonSesEmailNotification");
SendEmailResult sendEmailResult =
sesClient.sendEmail(
new SendEmailRequest()
.withSource(notificationContent.getSender().toString())
.withDestination(
new Destination(
Lists.newArrayList(
Arrays.stream(notificationContent.getRecipients())
.map(InternetAddress::toString)
.toList())))
.withMessage(
new Message(
new Content(notificationContent.getSubject()),
new Body().withHtml(new Content(notificationContent.getContent())))));
log.info("Email sent using Amazon SES. Message ID: {}", sendEmailResult.getMessageId());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
* Copyright 2023-2024 Broadcom
* SPDX-License-Identifier: Apache-2.0
*/

package com.vmware.taurus.service.notification;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.mail.*;
import javax.mail.internet.MimeMessage;
import java.util.Arrays;
import java.util.Properties;
import java.util.stream.Collectors;

/**
* The DefaultEmailNotification class is responsible for implementing the email notifications
* functionality. The single exposed send method accepts pre-build notification object.
*
* @see DataJobNotification
* @see NotificationContent
*/
public class DefaultEmailNotification implements EmailNotification {

static Logger log = LoggerFactory.getLogger(DefaultEmailNotification.class);

private static final String CONTENT_TYPE = "text/html; charset=utf-8";

private final Session session;
private final EmailConfiguration emailConfiguration;

public DefaultEmailNotification(EmailConfiguration emailConfiguration) {
this.emailConfiguration = emailConfiguration;
Properties properties = System.getProperties();
properties.putAll(emailConfiguration.smtpWithPrefix());
properties.setProperty("mail.transport.protocol", emailConfiguration.getTransportProtocol());
session = Session.getInstance(properties);
}

public void send(NotificationContent notificationContent) throws MessagingException {
log.debug("Send DefaultEmailNotification");
if (recipientsExist(notificationContent.getRecipients())) {
try {
sendEmail(notificationContent, notificationContent.getRecipients());
} catch (SendFailedException firstException) {
log.info(
"Following addresses are invalid: {}",
concatAddresses(firstException.getInvalidAddresses()));
var validUnsentAddresses = firstException.getValidUnsentAddresses();
if (recipientsExist(validUnsentAddresses)) {
log.info("Retrying unsent valid addresses: {}", concatAddresses(validUnsentAddresses));
try {
sendEmail(notificationContent, validUnsentAddresses);
} catch (SendFailedException retriedException) {
log.warn(
"\nwhat: {}\nwhy: {}\nconsequence: {}\ncountermeasure: {}",
"Unable to send notification to the recipients",
retriedException.getMessage(),
"Listed recipients won't receive notifications for the job's status",
"There might be misconfiguration or outage. Check error message for more details");
}
}
}
}
}

private void sendEmail(NotificationContent notificationContent, Address[] recipients)
throws MessagingException {
if (emailConfiguration.isAuthEnabled()) {
sendAuthenticatedEmail(notificationContent, recipients);
} else {
sendUnauthenticatedEmail(notificationContent, recipients);
}
}

private void sendUnauthenticatedEmail(
NotificationContent notificationContent, Address[] recipients) throws MessagingException {
var mimeMessage = prepareMessage(notificationContent);
Transport.send(mimeMessage, recipients);
}

private void sendAuthenticatedEmail(NotificationContent notificationContent, Address[] recipients)
throws MessagingException {
Transport transport = session.getTransport();
var mimeMessage = prepareMessage(notificationContent);
try {
transport.connect(emailConfiguration.getUsername(), emailConfiguration.getPassword());
transport.sendMessage(mimeMessage, recipients);
} finally {
transport.close();
}
}

private MimeMessage prepareMessage(NotificationContent notificationContent)
throws MessagingException {
MimeMessage mimeMessage = new MimeMessage(session);
mimeMessage.setFrom(notificationContent.getSender());
mimeMessage.setRecipients(MimeMessage.RecipientType.TO, notificationContent.getRecipients());
mimeMessage.setSubject(notificationContent.getSubject());
mimeMessage.setContent(notificationContent.getContent(), CONTENT_TYPE);
return mimeMessage;
}

private boolean recipientsExist(Address[] recipients) {
return recipients.length > 0;
}

String concatAddresses(Address[] addresses) {
return addresses == null
? null
: Arrays.stream(addresses).map(Address::toString).collect(Collectors.joining(" "));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@

package com.vmware.taurus.service.notification;

import java.util.HashMap;
import java.util.Map;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.HashMap;
import java.util.Map;

@Configuration
public class EmailConfiguration {

Expand All @@ -19,6 +20,15 @@ public class EmailConfiguration {
private final String MAIL_SMTP_AUTH = "mail.smtp.auth";
private final String TRANSPORT_PROTOCOL = "transport.protocol";

@Bean
public EmailNotification emailNotification() {
if (this.smtpWithPrefix().get("mail.smtp.user").startsWith("arn:aws:")) {
return new AmazonSesEmailNotification(this);
} else {
return new DefaultEmailNotification(this);
}
}

@Bean
@ConfigurationProperties(prefix = "mail")
Map<String, String> mail() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,7 @@

package com.vmware.taurus.service.notification;

import java.util.Arrays;
import java.util.Properties;
import java.util.stream.Collectors;
import javax.mail.Address;
import javax.mail.MessagingException;
import javax.mail.SendFailedException;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.MimeMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

/**
* The EmailNotification class is responsible for implementing the email notifications
Expand All @@ -25,94 +14,7 @@
* @see DataJobNotification
* @see NotificationContent
*/
@Component
public class EmailNotification {
public interface EmailNotification {

static Logger log = LoggerFactory.getLogger(EmailNotification.class);

private static final String CONTENT_TYPE = "text/html; charset=utf-8";

private final Session session;
private final EmailConfiguration emailConfiguration;

public EmailNotification(EmailConfiguration emailConfiguration) {
this.emailConfiguration = emailConfiguration;
Properties properties = System.getProperties();
properties.putAll(emailConfiguration.smtpWithPrefix());
properties.setProperty("mail.transport.protocol", emailConfiguration.getTransportProtocol());
session = Session.getInstance(properties);
}

public void send(NotificationContent notificationContent) throws MessagingException {
if (recipientsExist(notificationContent.getRecipients())) {
try {
sendEmail(notificationContent, notificationContent.getRecipients());
} catch (SendFailedException firstException) {
log.info(
"Following addresses are invalid: {}",
concatAddresses(firstException.getInvalidAddresses()));
var validUnsentAddresses = firstException.getValidUnsentAddresses();
if (recipientsExist(validUnsentAddresses)) {
log.info("Retrying unsent valid addresses: {}", concatAddresses(validUnsentAddresses));
try {
sendEmail(notificationContent, validUnsentAddresses);
} catch (SendFailedException retriedException) {
log.warn(
"\nwhat: {}\nwhy: {}\nconsequence: {}\ncountermeasure: {}",
"Unable to send notification to the recipients",
retriedException.getMessage(),
"Listed recipients won't receive notifications for the job's status",
"There might be misconfiguration or outage. Check error message for more details");
}
}
}
}
}

private void sendEmail(NotificationContent notificationContent, Address[] recipients)
throws MessagingException {
if (emailConfiguration.isAuthEnabled()) {
sendAuthenticatedEmail(notificationContent, recipients);
} else {
sendUnauthenticatedEmail(notificationContent, recipients);
}
}

private void sendUnauthenticatedEmail(
NotificationContent notificationContent, Address[] recipients) throws MessagingException {
var mimeMessage = prepareMessage(notificationContent);
Transport.send(mimeMessage, recipients);
}

private void sendAuthenticatedEmail(NotificationContent notificationContent, Address[] recipients)
throws MessagingException {
Transport transport = session.getTransport();
var mimeMessage = prepareMessage(notificationContent);
try {
transport.connect(emailConfiguration.getUsername(), emailConfiguration.getPassword());
transport.sendMessage(mimeMessage, recipients);
} finally {
transport.close();
}
}

private MimeMessage prepareMessage(NotificationContent notificationContent)
throws MessagingException {
MimeMessage mimeMessage = new MimeMessage(session);
mimeMessage.setFrom(notificationContent.getSender());
mimeMessage.setRecipients(MimeMessage.RecipientType.TO, notificationContent.getRecipients());
mimeMessage.setSubject(notificationContent.getSubject());
mimeMessage.setContent(notificationContent.getContent(), CONTENT_TYPE);
return mimeMessage;
}

private boolean recipientsExist(Address[] recipients) {
return recipients.length > 0;
}

String concatAddresses(Address[] addresses) {
return addresses == null
? null
: Arrays.stream(addresses).map(Address::toString).collect(Collectors.joining(" "));
}
public void send(NotificationContent notificationContent) throws MessagingException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public void setup() {

this.dataJobNotification =
new DataJobNotification(
new EmailNotification(emailPropertiesConfiguration),
new DefaultEmailNotification(emailPropertiesConfiguration),
"Example Name",
TEST_USER,
Collections.singletonList("cc@dummy.com"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public void setup() throws IOException {

this.dataJobNotification =
new DataJobNotification(
new EmailNotification(emailPropertiesConfiguration),
new DefaultEmailNotification(emailPropertiesConfiguration),
"Example Name",
"your_username@vmware.com",
Collections.singletonList("cc@dummy.com"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
@SpringBootTest(classes = ControlplaneApplication.class)
public class EmailNotificationTest {

@Autowired private EmailNotification emailNotification;
@Autowired private DefaultEmailNotification emailNotification;

@Test
public void testConcatAddresses_nullAddresses_shouldReturnNull() {
Expand Down