Skip to content

Commit bab3420

Browse files
author
Rustam Aliyev
committed
Add scrub support for index rebuilding. Fixes elasticinbox#40 
1 parent f4a5d8b commit bab3420

File tree

9 files changed

+346
-138
lines changed

9 files changed

+346
-138
lines changed

‎itests/pom.xml

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -126,17 +126,27 @@
126126
<artifactId>rest-assured</artifactId>
127127
<version>1.6.2</version>
128128
<scope>test</scope>
129-
</dependency>
130-
<dependency>
131-
<groupId>org.junit</groupId>
132-
<artifactId>com.springsource.org.junit</artifactId>
133-
<version>4.9.0</version>
129+
<exclusions>
130+
<exclusion>
131+
<groupId>org.hamcrest</groupId>
132+
<artifactId>hamcrest-core</artifactId>
133+
</exclusion>
134+
<exclusion>
135+
<groupId>org.hamcrest</groupId>
136+
<artifactId>hamcrest-library</artifactId>
137+
</exclusion>
138+
</exclusions>
139+
</dependency>
140+
<dependency>
141+
<groupId>junit</groupId>
142+
<artifactId>junit</artifactId>
143+
<version>4.11</version>
134144
<scope>test</scope>
135145
</dependency>
136146
<dependency>
137147
<groupId>org.hamcrest</groupId>
138148
<artifactId>hamcrest-all</artifactId>
139-
<version>1.1</version>
149+
<version>1.3</version>
140150
<scope>test</scope>
141151
</dependency>
142152

‎itests/src/test/java/com/elasticinbox/itests/RestV2IT.java

Lines changed: 153 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/**
2-
* Copyright (c) 2011-2012 Optimax Software Ltd.
2+
* Copyright (c) 2011-2013 Optimax Software Ltd.
33
* All rights reserved.
44
*
55
* Redistribution and use in source and binary forms, with or without
@@ -35,11 +35,14 @@
3535
import java.io.IOException;
3636
import java.io.InputStream;
3737
import java.text.SimpleDateFormat;
38+
import java.util.ArrayList;
3839
import java.util.Date;
3940
import java.util.HashMap;
41+
import java.util.List;
4042
import java.util.Map;
4143
import java.util.UUID;
4244

45+
import org.apache.commons.lang3.tuple.Pair;
4346
import org.junit.Test;
4447
import org.junit.runner.RunWith;
4548
import org.ops4j.pax.exam.junit.ExamReactorStrategy;
@@ -48,6 +51,7 @@
4851

4952
import com.elasticinbox.core.model.LabelConstants;
5053
import com.elasticinbox.core.model.LabelCounters;
54+
import com.elasticinbox.core.model.Labels;
5155
import com.elasticinbox.core.model.Marker;
5256
import com.elasticinbox.core.model.ReservedLabels;
5357
import com.google.common.io.ByteStreams;
@@ -352,7 +356,9 @@ public void messageAddRemoveLabelsMarkersTest() throws IOException
352356
pathParam("messageId", messageId.toString()).
353357
expect().
354358
statusCode(200).and().
355-
body("message.labels", hasItems(0, 1, labelId1, labelId2)).
359+
// TODO: uncomment when fixed http://code.google.com/p/rest-assured/issues/detail?id=169
360+
//body("message.labels", hasItems(0, 1, labelId1, labelId2)).
361+
body("message.labels", hasItems(0, 1, labelId2)).
356362
body("message.markers", hasItems(Marker.SEEN.toString().toUpperCase())).
357363
when().
358364
get(REST_PATH + "/mailbox/message/{messageId}");
@@ -411,8 +417,11 @@ public void messageBatchMofifyDeleteTest() throws IOException
411417
pathParam("labelId", ReservedLabels.ALL_MAILS.getId()).
412418
expect().
413419
statusCode(200).and().
414-
body(messageId1.toString() + ".labels", hasItems(0, 1, labelId1, labelId2)).
415-
body(messageId2.toString() + ".labels", hasItems(0, 1, labelId1, labelId2)).
420+
body(messageId1.toString() + ".labels", hasItems(0, 1, labelId2)).
421+
body(messageId2.toString() + ".labels", hasItems(0, 1, labelId2)).
422+
// TODO: uncomment when fixed http://code.google.com/p/rest-assured/issues/detail?id=169
423+
//body(messageId1.toString() + ".labels", hasItems(0, 1, labelId1, labelId2)).
424+
//body(messageId2.toString() + ".labels", hasItems(0, 1, labelId1, labelId2)).
416425
body(messageId1.toString() + ".markers", hasItems(Marker.SEEN.toString().toUpperCase())).
417426
body(messageId2.toString() + ".markers", hasItems(Marker.SEEN.toString().toUpperCase())).
418427
when().
@@ -444,7 +453,7 @@ public void messageAttachmentAndRawTest() throws IOException
444453

445454
// add message
446455
UUID messageId = addMessage(EMAIL_LARGE_ATT, ReservedLabels.INBOX.getId());
447-
long fileSize = getResourceSize(EMAIL_LARGE_ATT);
456+
//long fileSize = getResourceSize(EMAIL_LARGE_ATT);
448457

449458
// get attachment by part id
450459
given().
@@ -508,12 +517,20 @@ public void messageUpdateTest() throws IOException
508517
given().
509518
pathParam("messageId", messageId.toString()).
510519
pathParam("labelId1", ReservedLabels.IMPORTANT.getId()).
511-
pathParam("labelId2", ReservedLabels.STARRED.getId()).
512520
pathParam("marker1", Marker.SEEN.toString().toLowerCase()).
513521
expect().
514522
statusCode(204).
515523
when().
516-
put(REST_PATH + "/mailbox/message/{messageId}?addlabel={labelId1}&addlabel={labelId2}&addmarker={marker1}");
524+
put(REST_PATH + "/mailbox/message/{messageId}?addlabel={labelId1}&addmarker={marker1}");
525+
526+
// TODO: merge labelId2 to above request when fixed http://code.google.com/p/rest-assured/issues/detail?id=169
527+
given().
528+
pathParam("messageId", messageId.toString()).
529+
pathParam("labelId2", ReservedLabels.STARRED.getId()).
530+
expect().
531+
statusCode(204).
532+
when().
533+
put(REST_PATH + "/mailbox/message/{messageId}?addlabel={labelId2}");
517534

518535
// overwrite message
519536
InputStream fin = this.getClass().getResourceAsStream(EMAIL_REGULAR);
@@ -724,73 +741,35 @@ public void countersTest() throws IOException
724741
}
725742

726743
@Test
727-
public void mailboxCountersScrubTest() throws IOException
744+
public void mailboxScrubTest() throws IOException
728745
{
729746
initAccount();
747+
748+
Pair<Labels, Map<Integer, List<UUID>>> pair = populateMailbox();
749+
Labels labels = pair.getLeft();
750+
Map<Integer, List<UUID>> messages = pair.getRight();
751+
752+
// TODO: wipe off counters and indexes here. need to communicate with metadata store directly
730753

731-
Map<String, UUID> messages = new HashMap<String, UUID>();
732-
LabelCounters inboxCounters = new LabelCounters();
733-
LabelCounters notifCounters = new LabelCounters();
734-
LabelCounters trashCounters = new LabelCounters();
735-
LabelCounters spamCounters = new LabelCounters();
736-
737-
// INBOX: add 5 messages, mark 2 as unread
738-
messages.put("inbox1", addMessage(EMAIL_REGULAR, ReservedLabels.INBOX.getId()));
739-
messages.put("inbox2", addMessage(EMAIL_REGULAR, ReservedLabels.INBOX.getId()));
740-
messages.put("inbox3", addMessage(EMAIL_REGULAR, ReservedLabels.INBOX.getId()));
741-
messages.put("inbox4", addMessage(EMAIL_REGULAR, ReservedLabels.INBOX.getId()));
742-
messages.put("inbox5", addMessage(EMAIL_REGULAR, ReservedLabels.INBOX.getId()));
743-
markAsRead(messages.get("inbox4"));
744-
markAsRead(messages.get("inbox5"));
745-
inboxCounters.setTotalMessages(5L);
746-
inboxCounters.setUnreadMessages(3L);
747-
748-
// NOTIFICATIONS: add 3 messages, mark 1 as read
749-
messages.put("notif1", addMessage(EMAIL_REGULAR, ReservedLabels.NOTIFICATIONS.getId()));
750-
messages.put("notif2", addMessage(EMAIL_REGULAR, ReservedLabels.NOTIFICATIONS.getId()));
751-
messages.put("notif3", addMessage(EMAIL_REGULAR, ReservedLabels.NOTIFICATIONS.getId()));
752-
markAsRead(messages.get("notif2"));
753-
notifCounters.setTotalMessages(3L);
754-
notifCounters.setUnreadMessages(2L);
755-
756-
// SPAM: add 5 messages, keep all unread
757-
messages.put("spam1", addMessage(EMAIL_REGULAR, ReservedLabels.SPAM.getId()));
758-
messages.put("spam2", addMessage(EMAIL_REGULAR, ReservedLabels.SPAM.getId()));
759-
messages.put("spam3", addMessage(EMAIL_REGULAR, ReservedLabels.SPAM.getId()));
760-
messages.put("spam4", addMessage(EMAIL_REGULAR, ReservedLabels.SPAM.getId()));
761-
messages.put("spam5", addMessage(EMAIL_REGULAR, ReservedLabels.SPAM.getId()));
762-
spamCounters.setTotalMessages(5L);
763-
spamCounters.setUnreadMessages(5L);
764-
765-
// TRASH: add 4 messages, mark 2 as read
766-
messages.put("trash1", addMessage(EMAIL_REGULAR, ReservedLabels.TRASH.getId()));
767-
messages.put("trash2", addMessage(EMAIL_REGULAR, ReservedLabels.TRASH.getId()));
768-
messages.put("trash3", addMessage(EMAIL_REGULAR, ReservedLabels.TRASH.getId()));
769-
messages.put("trash4", addMessage(EMAIL_REGULAR, ReservedLabels.TRASH.getId()));
770-
markAsRead(messages.get("trash1"));
771-
markAsRead(messages.get("trash3"));
772-
trashCounters.setTotalMessages(4L);
773-
trashCounters.setUnreadMessages(2L);
774-
775-
// check label counters
754+
// check label counters before scrub
776755
expect().
777756
statusCode(200).and().
778757
body("'" + ReservedLabels.INBOX.getId() + "'.total",
779-
equalTo(inboxCounters.getTotalMessages().intValue())).
758+
equalTo(labels.getLabelCounters(ReservedLabels.INBOX.getId()).getTotalMessages().intValue())).
780759
body("'" + ReservedLabels.INBOX.getId() + "'.unread",
781-
equalTo(inboxCounters.getUnreadMessages().intValue())).
760+
equalTo(labels.getLabelCounters(ReservedLabels.INBOX.getId()).getUnreadMessages().intValue())).
782761
body("'" + ReservedLabels.NOTIFICATIONS.getId() + "'.total",
783-
equalTo(notifCounters.getTotalMessages().intValue())).
762+
equalTo(labels.getLabelCounters(ReservedLabels.NOTIFICATIONS.getId()).getTotalMessages().intValue())).
784763
body("'" + ReservedLabels.NOTIFICATIONS.getId() + "'.unread",
785-
equalTo(notifCounters.getUnreadMessages().intValue())).
764+
equalTo(labels.getLabelCounters(ReservedLabels.NOTIFICATIONS.getId()).getUnreadMessages().intValue())).
786765
body("'" + ReservedLabels.SPAM.getId() + "'.total",
787-
equalTo(spamCounters.getTotalMessages().intValue())).
766+
equalTo(labels.getLabelCounters(ReservedLabels.SPAM.getId()).getTotalMessages().intValue())).
788767
body("'" + ReservedLabels.SPAM.getId() + "'.unread",
789-
equalTo(spamCounters.getUnreadMessages().intValue())).
768+
equalTo(labels.getLabelCounters(ReservedLabels.SPAM.getId()).getUnreadMessages().intValue())).
790769
body("'" + ReservedLabels.TRASH.getId() + "'.total",
791-
equalTo(trashCounters.getTotalMessages().intValue())).
770+
equalTo(labels.getLabelCounters(ReservedLabels.TRASH.getId()).getTotalMessages().intValue())).
792771
body("'" + ReservedLabels.TRASH.getId() + "'.unread",
793-
equalTo(trashCounters.getUnreadMessages().intValue())).
772+
equalTo(labels.getLabelCounters(ReservedLabels.TRASH.getId()).getUnreadMessages().intValue())).
794773
when().
795774
get(REST_PATH + "/mailbox?metadata=true").asString();
796775

@@ -804,24 +783,112 @@ public void mailboxCountersScrubTest() throws IOException
804783
expect().
805784
statusCode(200).and().
806785
body("'" + ReservedLabels.INBOX.getId() + "'.total",
807-
equalTo(inboxCounters.getTotalMessages().intValue())).
786+
equalTo(labels.getLabelCounters(ReservedLabels.INBOX.getId()).getTotalMessages().intValue())).
808787
body("'" + ReservedLabels.INBOX.getId() + "'.unread",
809-
equalTo(inboxCounters.getUnreadMessages().intValue())).
788+
equalTo(labels.getLabelCounters(ReservedLabels.INBOX.getId()).getUnreadMessages().intValue())).
810789
body("'" + ReservedLabels.NOTIFICATIONS.getId() + "'.total",
811-
equalTo(notifCounters.getTotalMessages().intValue())).
790+
equalTo(labels.getLabelCounters(ReservedLabels.NOTIFICATIONS.getId()).getTotalMessages().intValue())).
812791
body("'" + ReservedLabels.NOTIFICATIONS.getId() + "'.unread",
813-
equalTo(notifCounters.getUnreadMessages().intValue())).
792+
equalTo(labels.getLabelCounters(ReservedLabels.NOTIFICATIONS.getId()).getUnreadMessages().intValue())).
814793
body("'" + ReservedLabels.SPAM.getId() + "'.total",
815-
equalTo(spamCounters.getTotalMessages().intValue())).
794+
equalTo(labels.getLabelCounters(ReservedLabels.SPAM.getId()).getTotalMessages().intValue())).
816795
body("'" + ReservedLabels.SPAM.getId() + "'.unread",
817-
equalTo(spamCounters.getUnreadMessages().intValue())).
796+
equalTo(labels.getLabelCounters(ReservedLabels.SPAM.getId()).getUnreadMessages().intValue())).
818797
body("'" + ReservedLabels.TRASH.getId() + "'.total",
819-
equalTo(trashCounters.getTotalMessages().intValue())).
798+
equalTo(labels.getLabelCounters(ReservedLabels.TRASH.getId()).getTotalMessages().intValue())).
820799
body("'" + ReservedLabels.TRASH.getId() + "'.unread",
821-
equalTo(trashCounters.getUnreadMessages().intValue())).
800+
equalTo(labels.getLabelCounters(ReservedLabels.TRASH.getId()).getUnreadMessages().intValue())).
822801
when().
823802
get(REST_PATH + "/mailbox?metadata=true").asString();
803+
804+
// scrub label indexes
805+
expect().
806+
statusCode(204).
807+
when().
808+
post(REST_PATH + "/mailbox/scrub/indexes");
809+
810+
//check indexes
811+
for (int labelId : new int[] {
812+
ReservedLabels.INBOX.getId(), ReservedLabels.NOTIFICATIONS.getId(),
813+
ReservedLabels.SPAM.getId(), ReservedLabels.TRASH.getId()} )
814+
{
815+
given().
816+
pathParam("labelId", labelId).
817+
expect().
818+
statusCode(200).and().
819+
body("", hasItems(asStringArray(messages.get(labelId)))).
820+
body("", hasSize(messages.get(labelId).size())).
821+
when().
822+
get(REST_PATH + "/mailbox/label/{labelId}");
823+
}
824+
}
825+
826+
/**
827+
* Populates mailbox with messages and returns expected counters and message IDs.
828+
*
829+
* @return
830+
* @throws IOException
831+
*/
832+
protected static Pair<Labels, Map<Integer, List<UUID>>> populateMailbox() throws IOException
833+
{
834+
Map<Integer, List<UUID>> messages = new HashMap<Integer, List<UUID>>();
835+
Labels labels = new Labels();
836+
LabelCounters inboxCounters = new LabelCounters();
837+
LabelCounters notifCounters = new LabelCounters();
838+
LabelCounters trashCounters = new LabelCounters();
839+
LabelCounters spamCounters = new LabelCounters();
840+
841+
// INBOX: add 5 messages, mark 2 as unread
842+
messages.put(ReservedLabels.INBOX.getId(), new ArrayList<UUID>());
843+
List<UUID> inboxMessages = messages.get(ReservedLabels.INBOX.getId());
844+
inboxMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.INBOX.getId()));
845+
inboxMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.INBOX.getId()));
846+
inboxMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.INBOX.getId()));
847+
inboxMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.INBOX.getId()));
848+
inboxMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.INBOX.getId()));
849+
markAsRead(inboxMessages.get(3));
850+
markAsRead(inboxMessages.get(4));
851+
inboxCounters.setTotalMessages(5L);
852+
inboxCounters.setUnreadMessages(3L);
853+
labels.incrementCounters(ReservedLabels.INBOX.getId(), inboxCounters);
854+
855+
// NOTIFICATIONS: add 3 messages, mark 1 as read
856+
messages.put(ReservedLabels.NOTIFICATIONS.getId(), new ArrayList<UUID>());
857+
List<UUID> notifMessages = messages.get(ReservedLabels.NOTIFICATIONS.getId());
858+
notifMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.NOTIFICATIONS.getId()));
859+
notifMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.NOTIFICATIONS.getId()));
860+
notifMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.NOTIFICATIONS.getId()));
861+
markAsRead(notifMessages.get(1));
862+
notifCounters.setTotalMessages(3L);
863+
notifCounters.setUnreadMessages(2L);
864+
labels.incrementCounters(ReservedLabels.NOTIFICATIONS.getId(), notifCounters);
865+
866+
// SPAM: add 5 messages, keep all unread
867+
messages.put(ReservedLabels.SPAM.getId(), new ArrayList<UUID>());
868+
List<UUID> spamMessages = messages.get(ReservedLabels.SPAM.getId());
869+
spamMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.SPAM.getId()));
870+
spamMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.SPAM.getId()));
871+
spamMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.SPAM.getId()));
872+
spamMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.SPAM.getId()));
873+
spamMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.SPAM.getId()));
874+
spamCounters.setTotalMessages(5L);
875+
spamCounters.setUnreadMessages(5L);
876+
labels.incrementCounters(ReservedLabels.SPAM.getId(), spamCounters);
877+
878+
// TRASH: add 4 messages, mark 2 as read
879+
messages.put(ReservedLabels.TRASH.getId(), new ArrayList<UUID>());
880+
List<UUID> trashMessages = messages.get(ReservedLabels.TRASH.getId());
881+
trashMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.TRASH.getId()));
882+
trashMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.TRASH.getId()));
883+
trashMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.TRASH.getId()));
884+
trashMessages.add(addMessage(EMAIL_REGULAR, ReservedLabels.TRASH.getId()));
885+
markAsRead(trashMessages.get(0));
886+
markAsRead(trashMessages.get(2));
887+
trashCounters.setTotalMessages(4L);
888+
trashCounters.setUnreadMessages(2L);
889+
labels.incrementCounters(ReservedLabels.TRASH.getId(), trashCounters);
824890

891+
return Pair.of(labels, messages);
825892
}
826893

827894
/**
@@ -913,4 +980,21 @@ private LabelCounters getCounters(String json, Integer labelId)
913980

914981
return lc;
915982
}
983+
984+
/**
985+
* Convert List<T> to String array by calling toString() on all elements.
986+
*
987+
* @param list
988+
* @return
989+
*/
990+
private static <T> String[] asStringArray(List<T> list)
991+
{
992+
String result[] = new String[list.size()];
993+
994+
for (int i = 0; i < list.size(); i++) {
995+
result[i] = list.get(i).toString();
996+
}
997+
998+
return result;
999+
}
9161000
}

‎modules/core/pom.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -361,13 +361,13 @@
361361
<dependency>
362362
<groupId>junit</groupId>
363363
<artifactId>junit</artifactId>
364-
<version>4.10</version>
364+
<version>4.11</version>
365365
<scope>test</scope>
366366
</dependency>
367367
<dependency>
368368
<groupId>org.hamcrest</groupId>
369369
<artifactId>hamcrest-all</artifactId>
370-
<version>1.1</version>
370+
<version>1.3</version>
371371
<scope>test</scope>
372372
</dependency>
373373
<dependency>

‎modules/core/src/main/java/com/elasticinbox/core/MessageDAO.java

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -229,10 +229,16 @@ public void removeLabel(Mailbox mailbox, Set<Integer> labelIds, UUID messageId)
229229
public void purge(Mailbox mailbox, Date age) throws IOException;
230230

231231
/**
232-
* Calculates counters for all labels bu scanning through all messages. Used
233-
* for scrub operation.
232+
* Scrub operation recalculates counters and optionally rebuilds indexes for
233+
* all labels by scanning through all messages in given mailbox.
234234
*
235+
* Recalculated counters are returned but not actually updated in metadata
236+
* store. Label indexes, on the other hand, are actually updated with
237+
* message IDs by this operation.
238+
*
239+
* @param mailbox
240+
* @param rebuildIndex
235241
* @return
236242
*/
237-
public Labels calculateCounters(Mailbox mailbox);
243+
public Labels scrub(Mailbox mailbox, boolean rebuildIndex);
238244
}

0 commit comments

Comments
 (0)