Skip to content

Commit a1bd6ad

Browse files
Gangisetty-AlekhyaRaju BeemreddySuryakumari Kolli
authored
Rebased code changes in EI-2.X to EI-3.X and added TTL indexes automatic updation. (#507)
* Logger info add for the received events * Enhancements on the below items. - Reduced the duplicate calls on subscription checks - Introduced ttl for event object map collection - Fixed Null pointer exceptions - Cache implemented for repeat handler collection. - Added log statements to print response time of the mongo and ER queries * Configurable paramter is introduced to restrict the default timeouts for the REST calls while triggering the notifications. Updated the code to handle the index creation only once. * Added loggers with additional info * Fixed test cases * Fixed the failure test cases * Dummu push to test the travis build * Travis build testing * Dummy push to test the travis build * Testing failed Fts * Testing failed test cases * Failure tests * Fix for waitlist queue issue * checking for testcase failures * dummy push * checking for itegration test failures * Checking for Integration test failures * dummy push * dummy push2 * checking for integration testcases * dummy push to check for IT failures * dummy push * Checking Integration tests * check for ft failures * Fixed EventToObjectMap handling and removed the configurable paramter "notification.httpRequest.timeout" * final check * fixed the comments * checking for testcase failures * dummy push * checking for itegration test failures * Checking for Integration test failures * dummy push * dummy push2 * checking for integration testcases * dummy push to check for IT failures * dummy push * Checking Integration tests * check for ft failures * final check * fixed the comments * final check * final push * Fixed all the comments * Updated version in pom file from 3.1.2 to 3.1.3 * Fixed the comments and modified documentation. Co-authored-by: Raju Beemreddy <raju.x.beemreddy@ericsson.com> Co-authored-by: Suryakumari Kolli <suryakumari.kolli@ericsson.com>
1 parent 37177d6 commit a1bd6ad

File tree

31 files changed

+449
-137
lines changed

31 files changed

+449
-137
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<modelVersion>4.0.0</modelVersion>
77
<groupId>com.github.ericsson</groupId>
88
<artifactId>eiffel-intelligence</artifactId>
9-
<version>3.1.2</version>
9+
<version>3.1.3</version>
1010
<packaging>war</packaging>
1111

1212
<parent>

‎src/functionaltests/java/com/ericsson/ei/encryption/EncryptionSteps.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ public void eiffelEventsAreSent() throws IOException, InterruptedException {
9494
LOGGER.debug("Sending Eiffel events.");
9595
List<String> eventNamesToSend = getEventNamesToSend();
9696
eventManager.sendEiffelEvents(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend);
97-
List<String> eventsIdList = eventManager.getEventsIdList(EIFFEL_EVENTS_JSON_PATH,
97+
List<String> eventsIdList = eventManager.getEventIdsList(EIFFEL_EVENTS_JSON_PATH,
9898
eventNamesToSend);
9999
List<String> missingEventIds = dbManager.verifyEventsInDB(eventsIdList, 0);
100100
String errorMessage = "The following events are missing in mongoDB: "

‎src/functionaltests/java/com/ericsson/ei/notifications/trigger/SubscriptionNotificationSteps.java‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@
1313
import java.util.Arrays;
1414
import java.util.List;
1515
import java.util.concurrent.TimeUnit;
16+
import java.util.stream.Collectors;
17+
import java.util.stream.Stream;
1618

19+
import org.bson.Document;
1720
import org.json.JSONArray;
1821
import org.json.JSONException;
1922
import org.junit.Ignore;
@@ -45,13 +48,15 @@
4548
import cucumber.api.java.en.Given;
4649
import cucumber.api.java.en.Then;
4750
import cucumber.api.java.en.When;
51+
import util.IntegrationTestBase;
4852

4953
@Ignore
5054
@TestPropertySource(properties = {
5155
"spring.data.mongodb.database: SubscriptionNotificationSteps",
5256
"failed.notifications.collection.name: SubscriptionNotificationSteps-failedNotifications",
5357
"rabbitmq.exchange.name: SubscriptionNotificationSteps-exchange",
54-
"rabbitmq.queue.suffix: SubscriptionNotificationSteps" })
58+
"rabbitmq.queue.suffix: SubscriptionNotificationSteps",
59+
"aggregations.collection.ttl: 0"})
5560
public class SubscriptionNotificationSteps extends FunctionalTestBase {
5661

5762
private static final Logger LOGGER = getLogger(SubscriptionNotificationSteps.class);
@@ -95,11 +100,12 @@ public class SubscriptionNotificationSteps extends FunctionalTestBase {
95100

96101
@Autowired
97102
private EmailSender emailSender;
98-
103+
99104
private SimpleSmtpServer smtpServer;
100105
private ClientAndServer restServer;
101106
private MockServerClient mockClient;
102107
private ResponseEntity response;
108+
private String aggregatedEventId;
103109

104110
@Before()
105111
public void beforeScenario() {
@@ -165,10 +171,16 @@ public void send_eiffel_events() throws Throwable {
165171
LOGGER.debug("About to send Eiffel events.");
166172
List<String> eventNamesToSend = getEventNamesToSend();
167173
eventManager.sendEiffelEvents(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend);
168-
List<String> missingEventIds = dbManager
169-
.verifyEventsInDB(eventManager.getEventsIdList(
170-
EIFFEL_EVENTS_JSON_PATH, eventNamesToSend),
171-
0);
174+
JsonNode parsedJSON = IntegrationTestBase.getJSONFromFile(EIFFEL_EVENTS_JSON_PATH);
175+
eventNamesToSend.forEach((String eventName) -> {
176+
JsonNode eventJson = parsedJSON.get(eventName);
177+
if (eventName.contains("EiffelArtifactCreatedEvent")) {
178+
aggregatedEventId = eventJson.get("meta").get("id").toString();
179+
}
180+
});
181+
aggregatedEventId = aggregatedEventId.substring(1,aggregatedEventId.length()-1);
182+
List<String> list = Stream.of(aggregatedEventId).collect(Collectors.toList());
183+
List<String> missingEventIds = dbManager.verifyEventsInDB(list,0);
172184
assertEquals("The following events are missing in mongoDB: " + missingEventIds.toString(),
173185
0,
174186
missingEventIds.size());
@@ -180,7 +192,7 @@ public void wait_for_ei_to_aggregate_objects() throws Throwable {
180192
List<String> eventNamesToSend = getEventNamesToSend();
181193
LOGGER.debug("Checking Aggregated Objects.");
182194
List<String> arguments = new ArrayList<>(
183-
eventManager.getEventsIdList(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend));
195+
eventManager.getEventIdsList(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend));
184196
arguments.add("id=TC5");
185197
arguments.add("conclusion=SUCCESSFUL");
186198
List<String> missingArguments = dbManager.verifyAggregatedObjectInDB(arguments);
@@ -487,4 +499,5 @@ private int getDbSizeForCondition(int minWaitTime, int maxWaitTime, int expected
487499

488500
return queryResult.size();
489501
}
502+
490503
}

‎src/functionaltests/java/com/ericsson/ei/notifications/ttl/TestTTLSteps.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@
2828
import com.ericsson.ei.exception.AuthenticationException;
2929
import com.ericsson.ei.exception.MongoDBConnectionException;
3030
import com.ericsson.ei.mongo.MongoCondition;
31+
import com.ericsson.ei.mongo.MongoConstants;
3132
import com.ericsson.ei.mongo.MongoDBHandler;
3233
import com.ericsson.ei.notifications.InformSubscriber;
34+
import com.ericsson.ei.subscription.SubscriptionHandler;
3335
import com.ericsson.ei.utils.FunctionalTestBase;
3436
import com.ericsson.ei.utils.HttpRequest;
3537
import com.ericsson.ei.utils.HttpRequest.HttpMethod;
@@ -91,6 +93,9 @@ public class TestTTLSteps extends FunctionalTestBase {
9193
@Autowired
9294
private InformSubscriber informSubscriber;
9395

96+
@Autowired
97+
private SubscriptionHandler subscriptionHandler;
98+
9499
@Before("@TestNotificationRetries")
95100
public void beforeScenario() {
96101
setUpMockServer();
@@ -104,7 +109,7 @@ public void afterScenario() throws IOException {
104109
}
105110

106111
@Given("^Subscription is created$")
107-
public void create_subscription_object() throws IOException, JSONException {
112+
public void create_subscription_object() throws IOException, JSONException, MongoDBConnectionException {
108113

109114
LOGGER.debug("Starting scenario @TestNotificationRetries.");
110115
mongoDBHandler.dropCollection(database, failedNotificationCollection);
@@ -115,6 +120,7 @@ public void create_subscription_object() throws IOException, JSONException {
115120
subscriptionStr = subscriptionStr.replaceAll("\\{port\\}", String.valueOf(clientAndServer.getPort()));
116121

117122
subscriptionObject = new ObjectMapper().readTree(subscriptionStr);
123+
mongoDBHandler.createTTLIndex(database, failedNotificationCollection, MongoConstants.TIME, 1);
118124
assertEquals(false, subscriptionObject.get("notificationMeta").toString().contains("{port}"));
119125
}
120126

@@ -166,7 +172,7 @@ public void eiffel_events_are_sent() throws Throwable {
166172
List<String> eventNamesToSend = getEventNamesToSend();
167173
eventManager.sendEiffelEvents(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend);
168174
List<String> missingEventIds = dbManager.verifyEventsInDB(
169-
eventManager.getEventsIdList(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend), 0);
175+
eventManager.getEventIdsList(EIFFEL_EVENTS_JSON_PATH, eventNamesToSend), 0);
170176
assertEquals("The following events are missing in mongoDB: " + missingEventIds.toString(), 0,
171177
missingEventIds.size());
172178
LOGGER.debug("Eiffel event is sent");
@@ -177,6 +183,8 @@ public void aggregated_object_is_created() throws Throwable {
177183
// verify that aggregated object is created and present in db
178184
LOGGER.debug("Checking presence of aggregated Object");
179185
List<String> allObjects = mongoDBHandler.getAllDocuments(database, collection);
186+
String id = eventManager.getEventIdsList(EIFFEL_EVENTS_JSON_PATH, getEventNamesToSend()).get(0);
187+
subscriptionHandler.checkSubscriptionForObject(allObjects.get(0), id);
180188
assertEquals(1, allObjects.size());
181189
}
182190

‎src/functionaltests/java/com/ericsson/ei/rules/RuleTestSteps.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,10 @@ public void get_content(String contentFileName) throws Throwable {
107107
if (expectedResponse instanceof JSONArray) {
108108
JSONArray expectedArray = new JSONArray(responseBody);
109109
JSONArray responseArray = new JSONArray(response.getBody().toString());
110+
for (int i = 0; i < responseArray.length(); i++) {
111+
JSONObject jsonobject = responseArray.getJSONObject(i);
112+
jsonobject.remove("Time");
113+
}
110114
assertEquals(expectedArray, responseArray, true);
111115
} else {
112116
JSONObject expectedObject = new JSONObject(responseBody);

‎src/functionaltests/java/com/ericsson/ei/threadingAndWaitlistRepeat/ThreadingAndWaitlistRepeatSteps.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public void event_to_object_map_is_manipulated_to_include_the_sent_events() thro
8080
List<String> eventNames = getEventNamesToSend();
8181
for (String eventName : eventNames) {
8282
JsonNode eventJson = parsedJSON.get(eventName);
83-
eventToObjectMapHanler.updateEventToObjectMapInMemoryDB(rulesObject, eventJson.toString(), dummyObjectID);
83+
eventToObjectMapHanler.updateEventToObjectMapInMemoryDB(rulesObject, eventJson.toString(), dummyObjectID, 0);
8484
}
8585
}
8686

‎src/functionaltests/java/com/ericsson/ei/utils/EventManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public void sendEiffelEvent(String eiffelEventJson) {
6060
* @return list of event IDs
6161
* @throws IOException
6262
*/
63-
public List<String> getEventsIdList(String eiffelEventsJsonPath, List<String> eventNames) throws IOException {
63+
public List<String> getEventIdsList(String eiffelEventsJsonPath, List<String> eventNames) throws IOException {
6464
List<String> eventsIdList = new ArrayList<>();
6565
JsonNode parsedJSON = getJSONFromFile(eiffelEventsJsonPath);
6666
for (String eventName : eventNames) {

‎src/integrationtests/java/com/ericsson/ei/integrationtests/flow/FlowStepsIT.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,12 @@ public class FlowStepsIT extends IntegrationTestBase {
8888
private JenkinsXmlData jenkinsXmlData;
8989
private SubscriptionObject subscriptionObject;
9090
private JSONObject jobStatusData;
91+
public String aggregatedEvent;
9192

9293
@Given("^the rules \"([^\"]*)\"$")
9394
public void rules(String rulesFilePath) throws Throwable {
9495
this.rulesFilePath = rulesFilePath;
96+
aggregatedEvent = getStartEvent(this.rulesFilePath);
9597
}
9698

9799
@Given("^the events \"([^\"]*)\"$")
@@ -167,7 +169,7 @@ public void conditionOrRequirementIsAddedInSubscription(String condition,
167169

168170
@When("^the eiffel events are sent$")
169171
public void eiffelEventsAreSent() throws Throwable {
170-
super.sendEventsAndConfirm();
172+
super.sendEventsAndConfirm(aggregatedEvent);
171173
}
172174

173175
@When("^the upstream input events are sent")

‎src/integrationtests/java/com/ericsson/ei/integrationtests/notification/FailedNotificationStepsIT.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ public class FailedNotificationStepsIT extends IntegrationTestBase {
5555
private String eventsFilePath;
5656
private ObjectMapper objectMapper = new ObjectMapper();
5757
private int extraEventsCount = 0;
58+
public String aggregatedEvent;
5859

5960
private SubscriptionObject subscriptionObject;
6061

@@ -64,6 +65,7 @@ public class FailedNotificationStepsIT extends IntegrationTestBase {
6465
@Given("^the rules \"([^\"]*)\"$")
6566
public void rules(String rulesFilePath) throws Throwable {
6667
this.rulesFilePath = rulesFilePath;
68+
aggregatedEvent = getStartEvent(this.rulesFilePath);
6769
}
6870

6971
@Given("^the events \"([^\"]*)\"$")
@@ -100,7 +102,7 @@ public void conditionOrRequirementIsAddedInSubscription(String condition,
100102

101103
@When("^the eiffel events are sent$")
102104
public void eiffelEventsAreSent() throws Throwable {
103-
super.sendEventsAndConfirm();
105+
super.sendEventsAndConfirm(aggregatedEvent);
104106
}
105107

106108
@When("^rest post body media type is set to \"([^\"]*)\" is set in subscription$")

‎src/integrationtests/java/util/IntegrationTestBase.java‎

Lines changed: 74 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,28 @@
1313
*/
1414
package util;
1515

16+
import static org.junit.Assert.assertEquals;
1617
import static org.junit.Assert.fail;
1718

1819
import java.io.File;
1920
import java.io.IOException;
21+
import java.io.InputStream;
22+
import java.net.URI;
2023
import java.net.URISyntaxException;
24+
import java.nio.charset.Charset;
2125
import java.util.Iterator;
2226
import java.util.List;
2327
import java.util.Map;
2428
import java.util.concurrent.TimeUnit;
29+
import java.util.stream.Collectors;
30+
import java.util.stream.Stream;
2531

2632
import javax.annotation.PostConstruct;
2733

2834
import org.apache.commons.io.FileUtils;
35+
import org.apache.commons.io.IOUtils;
36+
import org.json.JSONArray;
37+
import org.json.JSONObject;
2938
import org.skyscreamer.jsonassert.JSONAssert;
3039
import org.slf4j.Logger;
3140
import org.slf4j.LoggerFactory;
@@ -38,14 +47,11 @@
3847
import org.springframework.test.context.support.AbstractTestExecutionListener;
3948

4049
import com.ericsson.ei.mongo.MongoDBHandler;
50+
import com.ericsson.ei.mongo.MongoStringQuery;
4151
import com.ericsson.ei.utils.HttpRequest;
4252
import com.ericsson.ei.utils.HttpRequest.HttpMethod;
4353
import com.fasterxml.jackson.databind.JsonNode;
4454
import com.fasterxml.jackson.databind.ObjectMapper;
45-
import com.mongodb.MongoClient;
46-
import com.mongodb.client.MongoCollection;
47-
import com.mongodb.client.MongoDatabase;
48-
4955
import lombok.Setter;
5056

5157
public abstract class IntegrationTestBase extends AbstractTestExecutionListener {
@@ -95,7 +101,8 @@ public abstract class IntegrationTestBase extends AbstractTestExecutionListener
95101
private String failedNotificationCollectionName;
96102
@Value("${sessions.collection.name}")
97103
private String sessionsCollectionName;
98-
104+
105+
public String aggregatedEventId;
99106
/*
100107
* setFirstEventWaitTime: variable to set the wait time after publishing the first event. So any
101108
* thread looking for the events don't do it before actually populating events in the database
@@ -141,15 +148,17 @@ protected int extraEventsCount() {
141148
* @return
142149
* @throws Exception
143150
*/
144-
protected void sendEventsAndConfirm() throws Exception {
151+
protected void sendEventsAndConfirm(String aggregatedEvent) throws Exception {
145152
List<String> eventNames = getEventNamesToSend();
146153
int eventsCount = eventNames.size() + extraEventsCount();
147154

148155
JsonNode parsedJSON = getJSONFromFile(getEventsFilePath());
149-
150156
boolean alreadyExecuted = false;
151157
for (String eventName : eventNames) {
152158
JsonNode eventJson = parsedJSON.get(eventName);
159+
if (eventName.contains(aggregatedEvent)) {
160+
aggregatedEventId = eventJson.get("meta").get("id").toString();
161+
}
153162
String event = eventJson.toString();
154163

155164
rabbitTemplate.convertAndSend(event);
@@ -163,7 +172,6 @@ protected void sendEventsAndConfirm() throws Exception {
163172
*/
164173
TimeUnit.MILLISECONDS.sleep(DEFAULT_DELAY_BETWEEN_SENDING_EVENTS);
165174
}
166-
167175
waitForEventsToBeProcessed(eventsCount);
168176
checkResult(getCheckData());
169177
}
@@ -191,7 +199,7 @@ protected void sendEventsAndConfirm() throws Exception {
191199
*/
192200
protected abstract Map<String, JsonNode> getCheckData() throws IOException, Exception;
193201

194-
protected JsonNode getJSONFromFile(String filePath) throws Exception {
202+
public static JsonNode getJSONFromFile(String filePath) throws Exception {
195203
try {
196204
String expectedDocument = FileUtils.readFileToString(new File(filePath), "UTF-8");
197205
return objectMapper.readTree(expectedDocument);
@@ -206,7 +214,6 @@ protected JsonNode getJSONFromFile(String filePath) throws Exception {
206214

207215
/**
208216
* Wait for certain amount of events to be processed.
209-
*
210217
* @param eventsCount - An int which indicated how many events that should be processed.
211218
* @return
212219
* @throws InterruptedException
@@ -236,12 +243,65 @@ protected void waitForEventsToBeProcessed(int eventsCount) throws InterruptedExc
236243
* @return amount of processed events
237244
*/
238245
private long countProcessedEvents(String database, String collectionName) {
239-
MongoClient mongoClient = mongoDBHandler.getMongoClient();
240-
MongoDatabase db = mongoClient.getDatabase(database);
241-
MongoCollection collection = db.getCollection(collectionName);
242-
return collection.count();
246+
int count = 0;
247+
List<String> documents = null;
248+
String queryString = "{\"_id\": " + aggregatedEventId + "}";
249+
MongoStringQuery query = new MongoStringQuery(queryString);
250+
documents = mongoDBHandler.find(database, collectionName, query);
251+
JSONObject json = new JSONObject(documents.get(0));
252+
JSONArray jsonArray = new JSONArray();
253+
jsonArray.put(json.get("objects"));
254+
String objectId = json.get("objects").toString();
255+
for (int i = 0; i < objectId.length(); i++) {
256+
if (objectId.charAt(i) == ',')
257+
count++;
258+
}
259+
count++;
260+
return count;
261+
}
262+
263+
/**
264+
* StartEvent is Fetched from the given rules
265+
* @param rules file
266+
* @return StartEvent type
267+
*/
268+
public String getStartEvent(String rules) throws Exception {
269+
String content, type = null;
270+
content = readRulesFileFromPath(rules);
271+
JSONArray json = new JSONArray(content);
272+
if (content.isEmpty()) {
273+
throw new Exception("Rules content cannot be empty");
274+
} else {
275+
for (int i = 0; i < json.length(); i++) {
276+
JSONObject jsonObj = json.getJSONObject(i);
277+
if (jsonObj.get("StartEvent").equals("YES")) {
278+
type = jsonObj.get("Type").toString();
279+
break;
280+
}
281+
}
282+
}
283+
return type;
243284
}
244285

286+
/**
287+
* Reads the rules from given file path
288+
* @param rules
289+
* @return rules content
290+
* @throws IOException
291+
*/
292+
private String readRulesFileFromPath(String rules) throws IOException {
293+
String rulesJsonFileContent = null;
294+
try (InputStream inputStream = this.getClass().getResourceAsStream(rules)) {
295+
if (inputStream == null) {
296+
rulesJsonFileContent = FileUtils.readFileToString(new File(rules),
297+
Charset.defaultCharset());
298+
} else {
299+
rulesJsonFileContent = IOUtils.toString(inputStream, "UTF-8");
300+
}
301+
}
302+
return rulesJsonFileContent;
303+
}
304+
245305
/**
246306
* Retrieves the result from EI and checks if it equals the expected data
247307
*

0 commit comments

Comments
 (0)