20180604 jrse #156 (#167)

* #156: modified log entry.

mv|cp|save:src_ns,src_oid;metadata_key=metadata_value:metadata_key=metadata_value:....

* #156: basic doveadm rmb plugin

- basic integration of rmb plugin
- lsppols as first command.

* #156: added: doveadm rmb plugin commands

- doveadm rmb config create
- doveadm rmb config show
- doveadm rmb config update key=value

* #156: doveadm rmb plugin new commands:

- ls / get / set

* #156: doveadm rmb plugin

- ls mb, rename, delete

* #156: rmb plugin command

+ save_log

* #156: cleanup.

* #156: cleanup

-removed debug message

* #156: doveadm rmb save_log

- restores the index entry of moved objects.

* #156: doveadm rmb command check indices

- new doveadm rmb check indices -u <username> 
  iterates over all user mailboxes and checks if mail does exist.
  if mail does not exist, it prints out message with uid, guid and oid.

* #156: doveadm rmb check indices

- detect not referenced mail objects in user namespace and print them
  out.

* #147: doveadm rmb check indices delete_not_referenced_objects

- new option for rmb check indices to delete all mail objects of a
  user which are not referenced in a index file (all mailboxes).
- doveadm force-resync -u .. <box> is available to restore index
entries.

* #149: doveadm force-resync -u <user> INBOX 

- force-resync ignores the mailbox parameter like mdbox does and
performes does a force-resync for all user mailboxes.

* #149: doveadm force-rebuild

- uid is always generated and updated

* #156: removed some warnings / addional input param checks.

* #156: build 2.3 fix.

* #147: check indices no longer loads metadata by default

- doveadm rmb check indices does no loger load metadata by default
- stat is done async.

* build fix.

* doveadm mailbox delete

- delete mail objects asynchronous

* #163 new config option  rados_check_empty_mailboxes = true|false new doveadm rmb mailbox delete cmd

* #163: build fix 2.3, moved dovecot doevadm mailbox_delete function to rbox_src

* 163: #ifdefs for 2.2.21 build and removed some warnings

* #163: removed some debug messages

* added dovecad plugin lib10_doveadm_rbox_plugin.la

* cleanup mailbox before do copybox

.travis.yml

@@ -46,6 +46,7 @@ script:
 
 - docker exec build sh -c 'rm -r /usr/local/var/mail/rbox'
 - docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10110 profile=/root/pop3-profile.conf users=100 clients=10 error_quit secs=150 output=/var/log/imaptest.log'
+- docker exec build sh -c 'rm -r /usr/local/var/mail/rbox'
 - docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10143 users=100 clients=10 error_quit secs=60 copybox=INBOX.Drafts output=/var/log/imaptest.log'
 - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Error:\""'
 - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"failed:\""'

rpm/dovecot-ceph-plugin.spec

@@ -97,6 +97,8 @@ find %{buildroot}%{_libdir}/dovecot/ -type f -name \*.a  -delete
 %defattr(-,root,root)
 %dir %{_libdir}/dovecot
 %{_libdir}/dovecot/lib*.so*
+%dir %{_libdir}/dovecot/doveadm
+%{_libdir}/dovecot/doveadm/lib10_doveadm_rbox_plugin.so
 
 %files -n librmb0
 %defattr(-,root,root)

src/librmb/rados-dovecot-ceph-cfg-impl.h

@@ -40,7 +40,7 @@ class RadosDovecotCephCfgImpl : public RadosDovecotCephCfg {
 
   std::string &get_key_prefix_keywords() override { return dovecot_cfg.get_key_prefix_keywords(); }
   void update_metadata(const std::string &key, const char *value_) override { dovecot_cfg.update_metadata(key, value_); }
-
+  bool is_rbox_check_empty_mailboxes() override { return dovecot_cfg.is_rbox_check_empty_mailboxes(); }
   bool is_ceph_posix_bugfix_enabled() override { return dovecot_cfg.is_ceph_posix_bugfix_enabled(); }
   // rados config
   bool is_user_mapping() override { return rados_cfg.is_user_mapping(); }
@@ -94,6 +94,9 @@ class RadosDovecotCephCfgImpl : public RadosDovecotCephCfg {
   int read_object(const std::string &oid, librados::bufferlist *buffer) override { return rados_cfg.read_object(oid, buffer); }
   void set_io_ctx_namespace(const std::string &namespace_) override { rados_cfg.set_io_ctx_namespace(namespace_); }
 
+  RadosConfig *get_dovecot_cfg() { return &dovecot_cfg; }
+  RadosCephConfig *get_rados_ceph_cfg() { return &rados_cfg; }
+
  private:
   RadosConfig dovecot_cfg;
   RadosCephConfig rados_cfg;

src/librmb/rados-dovecot-ceph-cfg.h

@@ -66,6 +66,7 @@ class RadosDovecotCephCfg {
   virtual int save_object(const std::string &oid, librados::bufferlist &buffer) = 0;
   virtual int read_object(const std::string &oid, librados::bufferlist *buffer) = 0;
   virtual void set_io_ctx_namespace(const std::string &namespace_) = 0;
+  virtual bool is_rbox_check_empty_mailboxes() = 0;
 };
 
 } /* namespace librmb */

src/librmb/rados-dovecot-config.cpp

@@ -11,6 +11,9 @@
 
 #include "rados-dovecot-config.h"
 
+#include <iostream>
+#include <sstream>
+
 namespace librmb {
 
 std::string pool_name;
@@ -22,14 +25,16 @@ RadosConfig::RadosConfig()
       rados_username("rados_user_name"),
       prefix_keyword("k"),
       bugfix_cephfs_posix_hardlinks("rbox_bugfix_cephfs_21652"),
-      save_log("rados_save_log") {
+      save_log("rados_save_log"),
+      rbox_check_empty_mailboxes("rados_check_empty_mailboxes") {
   config[pool_name] = "mail_storage";
 
   config[rbox_cfg_object_name] = "rbox_cfg";
   config[rbox_cluster_name] = "ceph";
   config[rados_username] = "client.admin";
   config[bugfix_cephfs_posix_hardlinks] = "false";
   config[save_log] = "";
+  config[rbox_check_empty_mailboxes] = "false";
   is_valid = false;
 }
 
@@ -54,6 +59,18 @@ void RadosConfig::update_metadata(const std::string &key, const char *value_) {
   }
 }
 
+std::string RadosConfig::to_string() {
+  std::stringstream ss;
+  ss << "Dovecot configuration: (90-plugin.conf)" << std::endl;
+  ss << "  " << rbox_cfg_object_name << "=" << config[rbox_cfg_object_name] << std::endl;
+  ss << "  " << rbox_cluster_name << "=" << config[rbox_cluster_name] << std::endl;
+  ss << "  " << rados_username << "=" << config[rados_username] << std::endl;
+  ss << "  " << bugfix_cephfs_posix_hardlinks << "=" << config[bugfix_cephfs_posix_hardlinks] << std::endl;
+  ss << "  " << save_log << "=" << config[save_log] << std::endl;
+  ss << "  " << rbox_check_empty_mailboxes << "=" << config[rbox_check_empty_mailboxes] << std::endl;
+  return ss.str();
+}
+
 RadosConfig::~RadosConfig() {}
 
 } /* namespace librmb */

src/librmb/rados-dovecot-config.h

@@ -46,6 +46,10 @@ class RadosConfig {
     return config[bugfix_cephfs_posix_hardlinks].compare("true") == 0 ? true : false;
   }
   void set_rbox_cfg_object_name(const std::string &value) { config[rbox_cfg_object_name] = value; }
+  bool is_rbox_check_empty_mailboxes() {
+    return config[rbox_check_empty_mailboxes].compare("true") == 0 ? true : false;
+  }
+  std::string to_string();
 
  private:
   bool string_contains_key(const std::string &str, enum rbox_metadata_key key);
@@ -60,6 +64,7 @@ class RadosConfig {
   std::string prefix_keyword;
   std::string bugfix_cephfs_posix_hardlinks;
   std::string save_log;
+  std::string rbox_check_empty_mailboxes;
   bool is_valid;
 };
 

src/librmb/rados-mail-object.cpp

@@ -62,17 +62,22 @@ std::string RadosMailObject::to_string(const string &padding) {
   ostringstream ss;
 
   ss << endl;
-  ss << padding << "MAIL:   " << static_cast<char>(RBOX_METADATA_MAIL_UID) << "(uid)=" << uid << endl;
-  ss << padding << "        "
-     << "oid = " << oid << endl;
+  ss << padding << "MAIL:   ";
+  if (!uid.empty()) {
+    ss << static_cast<char>(RBOX_METADATA_MAIL_UID) << "(uid)=" << uid << endl;
+    ss << padding << "        ";
+  }
+  ss << "oid = " << oid << endl;
   string recv_time;
   if (RadosUtils::convert_time_t_to_str(ts, &recv_time) >= 0) {
     ss << padding << "        " << static_cast<char>(RBOX_METADATA_RECEIVED_TIME) << "(receive_time)=" << recv_time
        << "\n";
   } else {
-    ss << padding << "        " << static_cast<char>(RBOX_METADATA_RECEIVED_TIME) << "(receive_time)= INVALID DATE : '"
-       << recv_time_str << "'"
-       << "\n";
+    if (!recv_time_str.empty()) {
+      ss << padding << "        " << static_cast<char>(RBOX_METADATA_RECEIVED_TIME)
+         << "(receive_time)= INVALID DATE : '" << recv_time_str << "'"
+         << "\n";
+    }
   }
   string save_time;
   if (RadosUtils::convert_time_t_to_str(save_date_rados, &save_time) >= 0) {
@@ -82,18 +87,21 @@ std::string RadosMailObject::to_string(const string &padding) {
     ss << padding << "        "
        << "save_time= INVALID DATE '" << save_date_rados << "'\n";
   }
+
   ss << padding << "        " << static_cast<char>(RBOX_METADATA_PHYSICAL_SIZE) << "(phy_size)=" << p_size << " "
      << static_cast<char>(RBOX_METADATA_VIRTUAL_SIZE) << "(v_size) = " << v_size << " stat_size=" << object_size
      << endl;
-  ss << padding << "        " << static_cast<char>(RBOX_METADATA_MAILBOX_GUID) << "(mailbox_guid)=" << mailbox_guid
-     << endl;
-
+  if (!mailbox_guid.empty()) {
+    ss << padding << "        " << static_cast<char>(RBOX_METADATA_MAILBOX_GUID) << "(mailbox_guid)=" << mailbox_guid
+       << endl;
+  }
   if (mb_orig_name.length() > 0) {
     ss << padding << "        " << static_cast<char>(RBOX_METADATA_ORIG_MAILBOX)
        << "(mailbox_orig_name)=" << mb_orig_name << endl;
   }
-  ss << padding << "        " << static_cast<char>(RBOX_METADATA_GUID) << "(mail_guid)=" << mail_guid << endl;
-
+  if (!mail_guid.empty()) {
+    ss << padding << "        " << static_cast<char>(RBOX_METADATA_GUID) << "(mail_guid)=" << mail_guid << endl;
+  }
   if (rbox_version.length() > 0) {
     ss << padding << "        " << static_cast<char>(RBOX_METADATA_VERSION) << "(rbox_version): " << rbox_version
        << endl;

src/librmb/rados-metadata.h

@@ -48,6 +48,25 @@ class RadosMetadata {
       stream >> *t;
     }
   }
+  static bool from_string(const std::string& str, RadosMetadata* metadata) {
+    std::stringstream ss(str);
+    std::string item;
+    std::vector<std::string> token;
+    while (std::getline(ss, item, '=')) {
+      token.push_back(item);
+    }
+    if (token.size() != 2 || metadata == nullptr) {
+      return false;
+    }
+    metadata->key = token[0];
+    metadata->bl.append(token[1]);
+    return true;
+  }
+  std::string to_string() {
+    std::stringstream str;
+    str << key << "=" << bl.to_str();
+    return str.str();
+  }
 
  public:
   ceph::bufferlist bl;

src/librmb/rados-save-log.h

@@ -17,16 +17,66 @@
 #include <cstdio>
 #include <vector>
 #include <sstream>
+#include <list>
+#include <iostream>
+
+#include "rados-metadata.h"
 
 namespace librmb {
 
 class RadosSaveLogEntry {
  public:
   RadosSaveLogEntry() {}
   RadosSaveLogEntry(const std::string &oid_, const std::string &ns_, const std::string &pool_, const std::string &op_)
-      : oid(oid_), ns(ns_), pool(pool_), op(op_) {}
+      : oid(oid_), ns(ns_), pool(pool_), op(op_), metadata(0) {}
   ~RadosSaveLogEntry(){};
 
+  // format: mv|cp|save:src_ns,src_oid;metadata_key=metadata_value:metadata_key=metadata_value:....
+  //        e.g.: // mv:ns_src:src_oid;M=ABCDEFG:B=INBOX:U=1
+  bool parse_mv_op() {
+    int pos = op.find(";");
+    if (pos <= 0) {
+      return false;
+    }
+
+    std::stringstream left(op.substr(0, pos));
+    std::vector<std::string> left_tokens;
+    std::string item;
+    while (std::getline(left, item, ':')) {
+      left_tokens.push_back(item);
+    }
+    if (left_tokens.size() < 4) {
+      return false;
+    }
+
+    if (left_tokens[0].compare("mv") != 0) {
+      return true;  // not a move cmd.
+    }
+    // mv specific
+    src_ns = left_tokens[1];
+    src_oid = left_tokens[2];
+    src_user = left_tokens[3];
+    // parsing metadata.
+    std::stringstream right(op.substr(pos + 1, op.size()));
+    std::vector<std::string> right_tokens;
+    while (std::getline(right, item, ':')) {
+      librmb::RadosMetadata m;
+      if (!librmb::RadosMetadata::from_string(item, &m)) {
+        return false;
+      }
+      metadata.push_back(m);
+    }
+    return true;
+  }
+  static std::string op_save() { return "save"; }
+  static std::string op_cpy() { return "cpy"; }
+  static std::string op_mv(const std::string &src_ns, const std::string &src_oid, const std::string &src_user,
+                           std::list<librmb::RadosMetadata *> &metadata) {
+    std::stringstream mv;
+    mv << "mv:" << src_ns << ":" << src_oid << ":" << src_user << ";" << convert_metadata(metadata, ":");
+    return mv.str();
+  }
+
   friend std::ostream &operator<<(std::ostream &os, const RadosSaveLogEntry &obj) {
     os << obj.op << "," << obj.pool << "," << obj.ns << "," << obj.oid << std::endl;
     return os;
@@ -49,17 +99,40 @@ class RadosSaveLogEntry {
       obj.pool = csv_items[1];
       obj.ns = csv_items[2];
       obj.oid = csv_items[3];
+
+      obj.parse_mv_op();
+
     } else {
       is.setstate(std::ios::failbit);
     }
     return is;
   }
 
+  static std::string convert_metadata(std::list<librmb::RadosMetadata *> &metadata, const std::string &separator) {
+    std::stringstream metadata_str;
+    std::list<librmb::RadosMetadata *>::iterator list_it;
+    list_it = metadata.begin();
+    if (list_it != metadata.end()) {
+      metadata_str << (*list_it)->to_string();
+      list_it++;
+    }
+
+    for (; list_it != metadata.end(); list_it++) {
+      metadata_str << separator;
+      metadata_str << (*list_it)->to_string();
+    }
+    return metadata_str.str();
+  }
+
  public:
   std::string oid;   // oid
   std::string ns;    // namespace
   std::string pool;  // storage pool
   std::string op;    // operation: save, cp (copy), mv (move)
+  std::string src_oid;
+  std::string src_ns;
+  std::string src_user;
+  std::list<librmb::RadosMetadata> metadata;
 };
 
 class RadosSaveLog {

src/librmb/tools/rmb/Makefile.am

@@ -22,7 +22,7 @@ rmb_SOURCES = \
 	rmb-commands.cpp \
 	rmb-commands.h \
 	rmb.cpp \
-	rmb.h
+	rados-mail-box.h
 
 rmb_LDADD = $(shlibs)
 

src/librmb/tools/rmb/mailbox_tools.h

@@ -14,7 +14,7 @@
 
 #include <string>
 #include "rados-mail-object.h"
-#include "rmb.h"
+#include "rados-mail-box.h"
 
 namespace librmb {
 class MailboxTools {

src/librmb/tools/rmb/rmb.h → src/librmb/tools/rmb/rados-mail-box.h

@@ -22,6 +22,7 @@ namespace librmb {
 
 class RadosMailBox {
  public:
+
   RadosMailBox(const std::string& _mailbox_guid, int _mail_count, const std::string &_mbox_orig_name) : mailbox_guid(_mailbox_guid),
 													mail_count(_mail_count),
 													mbox_orig_name(_mbox_orig_name) {