Merge pull request #177 from ceph-dovecot/ls_orphaned_objects_#172

Ls orphaned objects #172

src/librmb/rados-mail-object.cpp

@@ -26,12 +26,9 @@ using librmb::RadosMailObject;
 const char RadosMailObject::X_ATTR_VERSION_VALUE[] = "0.1";
 const char RadosMailObject::DATA_BUFFER_NAME[] = "RADOS_MAIL_BUFFER";
 
-RadosMailObject::RadosMailObject() {
-  this->object_size = -1;
-  this->active_op = false;
-  this->save_date_rados = -1;
-  this->valid = true;
-}
+RadosMailObject::RadosMailObject()
+    : object_size(-1), active_op(false), save_date_rados(-1), valid(true), index_ref(false) {}
+
 RadosMailObject::~RadosMailObject() {}
 
 void RadosMailObject::set_guid(const uint8_t *_guid) { memcpy(this->guid, _guid, sizeof(this->guid)); }
@@ -65,7 +62,9 @@ std::string RadosMailObject::to_string(const string &padding) {
   if (!valid) {
     ss << padding << "<<<   MAIL OBJECT IS NOT VALID <<<<" << endl;
   }
-
+  if (!index_ref) {
+    ss << padding << "<<<   MAIL OBJECT HAS NO INDEX REFERENCE <<<<" << endl;
+  }
   ss << padding << "MAIL:   ";
   if (!uid.empty()) {
     ss << static_cast<char>(RBOX_METADATA_MAIL_UID) << "(uid)=" << uid << endl;

src/librmb/rados-mail-object.h

@@ -65,6 +65,8 @@ string get_metadata(const string& key) {
   return value;
 }
 
+bool is_index_ref() { return index_ref; }
+void set_index_ref(bool ref) { this->index_ref = ref; }
 bool is_valid() { return valid; }
 void set_valid(bool valid_) { valid = valid_; }
 bool has_active_op() { return active_op; }
@@ -95,10 +97,11 @@ const string get_extended_metadata(string& key) {
   map<string, ceph::bufferlist> attrset;
   map<string, ceph::bufferlist> extended_attrset;
   bool valid;
-
+  bool index_ref;
  public:
   static const char X_ATTR_VERSION_VALUE[];
   static const char DATA_BUFFER_NAME[];
+
 };
 
 }  // namespace librmb

src/librmb/tools/rmb/rmb.cpp

@@ -417,6 +417,9 @@ int main(int argc, const char **argv) {
     if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) {
       rmb_commands->load_objects(ms, mail_objects, sort_type);
       rmb_commands->query_mail_storage(&mail_objects, &parser, false, false);
+      std::cout << " NOTE: rmb tool does not have access to dovecot index. so all objects are set  <<<   MAIL OBJECT "
+                   "HAS NO INDEX REFERENCE <<<< use doveadm rmb ls - instead "
+                << std::endl;
     }
   } else if (opts.find("get") != opts.end()) {
     librmb::CmdLineParser parser(opts["get"]);

src/storage-rbox/doveadm-rbox-plugin.cpp

@@ -31,6 +31,8 @@ extern "C" {
 #include "istream.h"
 #include "doveadm-print.h"
 }
+#include <algorithm>
+
 #include "tools/rmb/rmb-commands.h"
 #include "rados-cluster.h"
 #include "rados-cluster-impl.h"
@@ -43,9 +45,7 @@ extern "C" {
 #include "rbox-save.h"
 #include "rbox-storage.hpp"
 
-#include <algorithm>
-
-
+int check_namespace_mailboxes(struct mail_namespace *ns, std::vector<librmb::RadosMailObject *> &mail_objects);
 
 class RboxDoveadmPlugin {
  public:
@@ -169,7 +169,7 @@ static int cmd_rmb_search_run(std::map<std::string, std::string> &opts, struct m
   int open = open_connection_load_config(&plugin);
   if (open < 0) {
     i_error("Error opening rados connection. Errorcode: %d", open);
-    return 0;
+    return open;
   }
 
   opts["namespace"] = user->username;
@@ -191,7 +191,12 @@ static int cmd_rmb_search_run(std::map<std::string, std::string> &opts, struct m
     delete ms;
     return ret;
   }
-
+  if (user->namespaces != NULL) {
+    struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces);
+    for (; ns != NULL; ns = ns->next) {
+      check_namespace_mailboxes(ns, mail_objects);
+    }
+  }
   if (download) {
     rmb_cmds.set_output_path(&parser);
   }
@@ -221,7 +226,17 @@ static int cmd_rmb_ls_run(struct doveadm_mail_cmd_context *ctx, struct mail_user
 
   if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) {
     std::vector<librmb::RadosMailObject *> mail_objects;
+
     ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false);
+
+    // TODO: check for mails without reference
+    auto it_mail = std::find_if(mail_objects.begin(), mail_objects.end(),
+                                [](librmb::RadosMailObject const *n) -> bool { return !n->is_index_ref(); });
+
+    if (it_mail != mail_objects.end()) {
+      std::cout << "There are unreference objects " << std::endl;
+    }
+
     for (auto mo : mail_objects) {
       delete mo;
     }
@@ -240,9 +255,6 @@ static int cmd_rmb_ls_mb_run(struct doveadm_mail_cmd_context *ctx, struct mail_u
   if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) {
     std::vector<librmb::RadosMailObject *> mail_objects;
     ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false);
-    for (auto mo : mail_objects) {
-      delete mo;
-    }
   } else {
     i_error("invalid ls search query");
     ctx->exit_code = -1;
@@ -654,11 +666,12 @@ static int iterate_mailbox(struct mail_namespace *ns, const struct mailbox_info
                                 [oid](librmb::RadosMailObject *m) { return m->get_oid().compare(oid) == 0; });
 
     if (it_mail == mail_objects.end()) {
-      std::cout << "   missing mail object: uid=" << mail->uid << " guid=" << guid << " oid : " << oid
-                << " available: " << (it_mail != mail_objects.end()) << std::endl;
+      /*  std::cout << "   missing mail object: uid=" << mail->uid << " guid=" << guid << " oid : " << oid
+                  << " available: " << (it_mail != mail_objects.end()) << std::endl;*/
       ++mail_count_missing;
     } else {
-      mail_objects.erase(it_mail);  // calls destructor of RadosMailObject*
+      (*it_mail)->set_index_ref(true);
+      // mail_objects.erase(it_mail);  // calls destructor of RadosMailObject*
     }
   }
   if (mailbox_search_deinit(&search_ctx) < 0) {
@@ -681,7 +694,7 @@ static int check_namespace_mailboxes(struct mail_namespace *ns, std::vector<libr
   struct mailbox_list_iterate_context *iter;
   const struct mailbox_info *info;
   int ret = 0;
-
+  std::cout << "INDEX: Check" << std::endl;
   iter = mailbox_list_iter_init(ns->list, "*", static_cast<enum mailbox_list_iter_flags>(
                                                    MAILBOX_LIST_ITER_RAW_LIST | MAILBOX_LIST_ITER_RETURN_NO_FLAGS));
   while ((info = mailbox_list_iter_next(iter)) != NULL) {
@@ -717,7 +730,11 @@ static int cmd_rmb_check_indices_run(struct doveadm_mail_cmd_context *ctx, struc
     check_namespace_mailboxes(ns, mail_objects);
   }
 
-  if (mail_objects.size() > 0) {
+  // TODO: check for mails with
+  auto it_mail = std::find_if(mail_objects.begin(), mail_objects.end(),
+                              [](librmb::RadosMailObject *m) { return !m->is_index_ref(); });
+
+  if (it_mail != mail_objects.end()) {
     std::cout << std::endl << "There are mail objects without a index reference: " << std::endl;
     std::cout << "NOTE: you can fix(restore) the lost index entries by using doveadm force-resync or delete the "
                  "unrefrenced objects from objectstore "
@@ -750,7 +767,7 @@ static int cmd_rmb_check_indices_run(struct doveadm_mail_cmd_context *ctx, struc
 
   for (auto mo : mail_objects) {
     std::cout << mo->to_string("  ") << std::endl;
-    if (open >= 0 && ctx_->delete_not_referenced_objects) {
+    if (open >= 0 && ctx_->delete_not_referenced_objects && !mo->is_index_ref()) {
       std::cout << "mail object: " << mo->get_oid().c_str()
                 << " deleted: " << (plugin.storage->delete_mail(mo) < 0 ? " FALSE " : " TRUE") << std::endl;
       ctx->exit_code = 2;