Skip to content

Commit

Permalink
Optimization by moving
Browse files Browse the repository at this point in the history 
- flags search from internal state
- date search from internal state
It alleviates de DB stress and increasing speed.
  • Loading branch information
@cncioranu
cncioranu committed May 4, 2020
1 parent 2f4595c commit 19525ad
Show file tree
Hide file tree
Showing 2 changed files with 237 additions and 43 deletions.
241 changes: 198 additions & 43 deletions src/dm_mailbox.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1217,6 +1217,7 @@ static gboolean _do_sort(GNode *node, DbmailMailbox *self)

return FALSE;
}

static GTree * mailbox_search(DbmailMailbox *self, search_key *s)
{
TRACE(TRACE_DEBUG, "Call: mailbox_search");
Expand All @@ -1228,6 +1229,14 @@ static GTree * mailbox_search(DbmailMailbox *self, search_key *s)
Connection_T c; ResultSet_T r; PreparedStatement_T st;
GTree *ids;
volatile char *inset = NULL;
/* helper for some operations in TREE mode search */
char *cond=NULL;
cond=malloc(30);
memset(cond,0,30);
int IST_DATE_COND =0 ;
/* if the query will be performed in sql mode */
int sql;
sql=1;

GString *t;
String_T q;
Expand All @@ -1243,6 +1252,8 @@ static GTree * mailbox_search(DbmailMailbox *self, search_key *s)
c = db_con_get();
t = g_string_new("");
q = p_string_new(self->pool, "");
s->found = g_tree_new_full((GCompareDataFunc)ucmpdata,NULL,(GDestroyNotify)uint64_free, (GDestroyNotify)uint64_free);

TRY
switch (s->type) {
case IST_HDRDATE_ON:
Expand Down Expand Up @@ -1342,22 +1353,6 @@ static GTree * mailbox_search(DbmailMailbox *self, search_key *s)
db_stmt_set_str(st, 4, partial);

break;

case IST_IDATE:
p_string_printf(q, "SELECT message_idnr FROM %smessages m "
"LEFT JOIN %sphysmessage p ON m.physmessage_id=p.id "
"WHERE mailbox_idnr = ? AND status < ? "
"%s "
"AND %s "
"ORDER BY message_idnr",
DBPFX, DBPFX,
inset?inset:"",
s->search);

st = db_stmt_prepare(c, p_string_str(q));
db_stmt_set_u64(st, 1, dbmail_mailbox_get_id(self));
db_stmt_set_int(st, 2, MESSAGE_STATUS_DELETE);
break;

case IST_DATA_BODY:
g_string_printf(t,db_get_sql(SQL_ENCODE_ESCAPE), "p.data");
Expand Down Expand Up @@ -1427,41 +1422,200 @@ static GTree * mailbox_search(DbmailMailbox *self, search_key *s)
db_stmt_set_str(st, 3, s->search);

break;

case IST_IDATE:
/* implemented search by state, avoiding queries in DB */
/*p_string_printf(q, "SELECT message_idnr FROM %smessages m "
"LEFT JOIN %sphysmessage p ON m.physmessage_id=p.id "
"WHERE mailbox_idnr = ? AND status < ? "
"%s "
"AND %s "
"ORDER BY message_idnr",
DBPFX, DBPFX,
inset?inset:"",
s->search);
st = db_stmt_prepare(c, p_string_str(q));
db_stmt_set_u64(st, 1, dbmail_mailbox_get_id(self));
db_stmt_set_int(st, 2, MESSAGE_STATUS_DELETE);
*/
//break;
default:
p_string_printf(q, "SELECT message_idnr FROM %smessages "
"WHERE mailbox_idnr = ? AND status < ? AND %s "
"ORDER BY message_idnr", DBPFX,
s->search); // FIXME: Sometimes s->search is ""

st = db_stmt_prepare(c, p_string_str(q));
db_stmt_set_u64(st, 1, dbmail_mailbox_get_id(self));
db_stmt_set_int(st, 2, MESSAGE_STATUS_DELETE);

if (strstr(s->search,"p.internal_date")){
/* the equivalent in query is at IST_IDATE, see above */
if (strstr(s->search,">")){

sql=0;
IST_DATE_COND=14;
strcpy(cond, strstr(s->search,">")+1);
TRACE(TRACE_DEBUG,"IST_IDATE [>]");
}
if (strstr(s->search,"=")){
sql=0;
IST_DATE_COND=15;
strcpy(cond, strstr(s->search,"=")+1);
TRACE(TRACE_DEBUG,"IST_IDATE [=]");
}
if (strstr(s->search,"<")){
sql=0;
IST_DATE_COND=16;
strcpy(cond, strstr(s->search,"<")+1);
TRACE(TRACE_DEBUG,"IST_IDATE [<]");
}
if (IST_DATE_COND!=0){
p_trim(cond," '><="); //TRIMMING
TRACE(TRACE_DEBUG,"IST_IDATE %s -> %s ", s->search,cond);
}

}

if (strcmp(s->search,"answered_flag=1")==0){
sql=0;
IST_DATE_COND=1;
}
if (strcmp(s->search,"deleted_flag=1")==0){
sql=0;
IST_DATE_COND=2;
}
if (strcmp(s->search,"flagged_flag=1")==0){
sql=0;
IST_DATE_COND=3;
}
if (strcmp(s->search,"recent_flag=1")==0){
sql=0;
IST_DATE_COND=4;
}
if (strcmp(s->search,"seen_flag=1")==0){
sql=0;
IST_DATE_COND=5;
}
if (strcmp(s->search,"draft_flag=1")==0){
sql=0;
IST_DATE_COND=6;
}
if (strcmp(s->search,"(seen_flag=0 AND recent_flag=1)")==0){
sql=0;
IST_DATE_COND=7;
}
if (strcmp(s->search,"recent_flag=0")==0){
sql=0;
IST_DATE_COND=8;
}
if (strcmp(s->search,"answered_flag=0")==0){
sql=0;
IST_DATE_COND=9;
}
if (strcmp(s->search,"deleted_flag=0")==0){
sql=0;
IST_DATE_COND=10;
}
if (strcmp(s->search,"flagged_flag=0")==0){
sql=0;
IST_DATE_COND=11;
}
if (strcmp(s->search,"seen_flag=0")==0){
sql=0;
IST_DATE_COND=12;
}
if (strcmp(s->search,"draft_flag=0")==0){
sql=0;
IST_DATE_COND=13;
}
//TRACE(TRACE_DEBUG,"IST_IDATE %s -> %d %d", s->search, sql,IST_DATE_COND);
if (sql==0){
int foundItems=0;
ids = MailboxState_getIds(self->mbstate);
GList *uids = g_tree_keys(ids);
uids = g_list_first(uids);
while (uids) {
uint64_t id = *(uint64_t *)uids->data;
if (! (w = g_tree_lookup(ids, &id))) {
TRACE(TRACE_ERR, "key missing in ids: [%" PRIu64 "]", id);
if (! g_list_next(uids)) break;
uids = g_list_next(uids);
continue;
}else{
MessageInfo *msginfo = g_tree_lookup(MailboxState_getMsginfo(self->mbstate), &id);

int found=0;
switch(IST_DATE_COND){
case 1: found=(int)(msginfo->flags[IMAP_FLAG_ANSWERED]==1);break;
case 2: found=(int)(msginfo->flags[IMAP_FLAG_DELETED]==1);break;
case 3: found=(int)(msginfo->flags[IMAP_FLAG_FLAGGED]==1);break;
case 4: found=(int)(msginfo->flags[IMAP_FLAG_RECENT]==1);break;
case 5: found=(int)(msginfo->flags[IMAP_FLAG_SEEN]==1);break;
case 6: found=(int)(msginfo->flags[IMAP_FLAG_DRAFT]==1);break;
case 7: found=(int)(msginfo->flags[IMAP_FLAG_SEEN]==0 && msginfo->flags[IMAP_FLAG_RECENT]==1);break;
case 8: found=(int)(msginfo->flags[IMAP_FLAG_RECENT]==0);break;
case 9: found=(int)(msginfo->flags[IMAP_FLAG_ANSWERED]==0);break;
case 10: found=(int)(msginfo->flags[IMAP_FLAG_DELETED]==0);break;
case 11: found=(int)(msginfo->flags[IMAP_FLAG_FLAGGED]==0);break;
case 12: found=(int)(msginfo->flags[IMAP_FLAG_SEEN]==0);break;
case 13: found=(int)(msginfo->flags[IMAP_FLAG_DRAFT]==0);break;
case 14: found=(int)(strcmp(msginfo->internaldate,cond)>0);break;
case 15: found=(int)(strcmp(msginfo->internaldate,cond)==0);break;
case 16: found=(int)(strcmp(msginfo->internaldate,cond)<0);break;
}
if (found==1){
//TRACE(TRACE_DEBUG, "Found (%d) %ld vs %ld ",IST_DATE_COND,id,msginfo->uid);
k = mempool_pop(small_pool, sizeof(uint64_t));
v = mempool_pop(small_pool, sizeof(uint64_t));
*k = id;
*v = *w;

g_tree_insert(s->found, k, v);
foundItems++;
}
if (! g_list_next(uids)) break;
uids = g_list_next(uids);
}
}
g_list_free(g_list_first(uids));
if (cond){
TRACE(TRACE_DEBUG,"IST_IDATE TREE found %s (%s)-> %d, found %d",
s->search,
cond,
IST_DATE_COND,
foundItems);
}else{
TRACE(TRACE_DEBUG,"IST_IDATE TREE found %s -> %d, found %d", s->search,IST_DATE_COND, foundItems);
}
}else{
p_string_printf(q, "SELECT message_idnr FROM %smessages "
"WHERE mailbox_idnr = ? AND status < ? AND %s "
"ORDER BY message_idnr", DBPFX,
s->search); // FIXME: Sometimes s->search is ""

st = db_stmt_prepare(c, p_string_str(q));
db_stmt_set_u64(st, 1, dbmail_mailbox_get_id(self));
db_stmt_set_int(st, 2, MESSAGE_STATUS_DELETE);
}
break;

}
if (st && sql==1){
int foundItems=0;
r = db_stmt_query(st);


r = db_stmt_query(st);

s->found = g_tree_new_full((GCompareDataFunc)ucmpdata,NULL,(GDestroyNotify)uint64_free, (GDestroyNotify)uint64_free);
ids = MailboxState_getIds(self->mbstate);
while (db_result_next(r)) {
id = db_result_get_u64(r,0);
if (! (w = g_tree_lookup(ids, &id))) {
TRACE(TRACE_ERR, "key missing in ids: [%" PRIu64 "]\n", id);
continue;
}
assert(w);

ids = MailboxState_getIds(self->mbstate);
while (db_result_next(r)) {
id = db_result_get_u64(r,0);
if (! (w = g_tree_lookup(ids, &id))) {
TRACE(TRACE_ERR, "key missing in ids: [%" PRIu64 "]\n", id);
continue;
k = mempool_pop(small_pool, sizeof(uint64_t));
v = mempool_pop(small_pool, sizeof(uint64_t));
*k = id;
*v = *w;
g_tree_insert(s->found, k, v);
foundItems++;
}
assert(w);

k = mempool_pop(small_pool, sizeof(uint64_t));
v = mempool_pop(small_pool, sizeof(uint64_t));
*k = id;
*v = *w;

g_tree_insert(s->found, k, v);
TRACE(TRACE_DEBUG,"IST_IDATE SQL found %s, found %d", s->search, foundItems);
}

if (s->type == IST_UNKEYWORD) {
GTree *old = NULL;
GTree *invert = g_tree_new_full((GCompareDataFunc)ucmpdata,NULL,(GDestroyNotify)uint64_free, (GDestroyNotify)uint64_free);
Expand Down Expand Up @@ -1517,6 +1671,7 @@ static GTree * mailbox_search(DbmailMailbox *self, search_key *s)

p_string_free(q,TRUE);
g_string_free(t,TRUE);
free(cond);

return s->found;
}
Expand Down
39 changes: 39 additions & 0 deletions src/dm_string.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,4 +211,43 @@ char * p_string_free(T S, gboolean free_block)
return s;
}


char *p_ltrim(char *str, const char *seps)
{
size_t totrim;
if (seps == NULL) {
seps = "\t\n\v\f\r ";
}
totrim = strspn(str, seps);
if (totrim > 0) {
size_t len = strlen(str);
if (totrim == len) {
str[0] = '\0';
}
else {
memmove(str, str + totrim, len + 1 - totrim);
}
}
return str;
}

char *p_rtrim(char *str, const char *seps)
{
int i;
if (seps == NULL) {
seps = "\t\n\v\f\r ";
}
i = strlen(str) - 1;
while (i >= 0 && strchr(seps, str[i]) != NULL) {
str[i] = '\0';
i--;
}
return str;
}

char *p_trim(char *str, const char *seps)
{
return p_ltrim(p_rtrim(str, seps), seps);
}

#undef T

0 comments on commit 19525ad

Please sign in to comment.