#148: save log file integration. changed format of log file to operat…

…ion,poolname,namespace,oid (operations: save,cp,mv). activate login in 90-plugin.conf: plugin { rados_save_log=logfile.log }

src/librmb/rados-dovecot-ceph-cfg-impl.h

@@ -33,7 +33,7 @@ class RadosDovecotCephCfgImpl : public RadosDovecotCephCfg {
   const std::string &get_rados_cluster_name() { return dovecot_cfg.get_rbox_cluster_name(); }
   const std::string &get_rados_username() { return dovecot_cfg.get_rados_username(); }
   void update_pool_name_metadata(const char *value) { dovecot_cfg.update_pool_name_metadata(value); }
-
+  const std::string &get_rados_save_log_file() { return dovecot_cfg.get_rados_save_log_file(); }
   const std::string &get_pool_name_metadata_key() { return dovecot_cfg.get_pool_name_metadata_key(); }
 
   std::string &get_pool_name() { return dovecot_cfg.get_pool_name(); }

src/librmb/rados-dovecot-ceph-cfg.h

@@ -24,6 +24,7 @@ class RadosDovecotCephCfg {
   // dovecot configuration
   virtual const std::string &get_rados_cluster_name() = 0;
   virtual const std::string &get_rados_username() = 0;
+  virtual const std::string &get_rados_save_log_file() = 0;
   virtual bool is_mail_attribute(enum rbox_metadata_key key) = 0;
   virtual bool is_updateable_attribute(enum rbox_metadata_key key) = 0;
   virtual void set_update_attributes(const std::string &update_attributes_) = 0;

src/librmb/rados-dovecot-config.h

@@ -31,7 +31,7 @@ class RadosConfig {
   std::map<std::string, std::string> *get_config() { return &config; }
 
   std::string &get_pool_name() { return config[pool_name]; }
-
+  const std::string &get_rados_save_log_file() { return config[save_log]; }
   bool is_config_valid() { return is_valid; }
   void set_config_valid(bool is_valid_) { this->is_valid = is_valid_; }
 

src/librmb/rados-save-log.cpp

@@ -14,19 +14,19 @@
 namespace librmb {
 
 bool RadosSaveLog::open() {
-  if (this->log_active) {
+  if (this->log_active && !ofs.is_open()) {
     ofs.open(this->logfile, std::ofstream::out | std::ofstream::app);
-    return ofs.is_open();
+    this->log_active = ofs.is_open();
   }
-  return true;
+  return this->log_active;
 }
 void RadosSaveLog::append(const RadosSaveLogEntry &entry) {
-  if (this->log_active) {
+  if (this->log_active && ofs.is_open()) {
     ofs << entry;
   }
 }
 bool RadosSaveLog::close() {
-  if (this->log_active) {
+  if (this->log_active && ofs.is_open()) {
     ofs.close();
     return !ofs.is_open();
   }

src/librmb/rados-save-log.h

@@ -19,15 +19,16 @@ namespace librmb {
 class RadosSaveLogEntry {
  public:
   RadosSaveLogEntry() {}
-  RadosSaveLogEntry(const std::string &oid_, const std::string &ns_, const std::string &pool_) {
+  RadosSaveLogEntry(const std::string &oid_, const std::string &ns_, const std::string &pool_, const std::string &op_) {
     this->oid = oid_;
     this->ns = ns_;
     this->pool = pool_;
+    this->op = op_;
   }
   ~RadosSaveLogEntry(){};
 
   friend std::ostream &operator<<(std::ostream &os, const RadosSaveLogEntry &obj) {
-    os << obj.oid << "," << obj.ns << "," << obj.pool << std::endl;
+    os << obj.op << "," << obj.pool << "," << obj.ns << "," << obj.oid << std::endl;
     return os;
   }
 
@@ -38,12 +39,13 @@ class RadosSaveLogEntry {
     std::vector<std::string> csv_items{std::sregex_token_iterator(item.begin(), item.end(), re, 1),
                                        std::sregex_token_iterator()};
     // read obj from stream
-    if (csv_items.size() < 3) {
+    if (csv_items.size() < 4) {
       is.setstate(std::ios::failbit);
     } else {
-      obj.oid = csv_items[0];
-      obj.ns = csv_items[1];
-      obj.pool = csv_items[2];
+      obj.op = csv_items[0];
+      obj.pool = csv_items[1];
+      obj.ns = csv_items[2];
+      obj.oid = csv_items[3];
     }
     return is;
   }
@@ -52,6 +54,7 @@ class RadosSaveLogEntry {
   std::string oid;   // oid
   std::string ns;    // namespace
   std::string pool;  // storage pool
+  std::string op;    // operation: save, cp (copy), mv (move)
 };
 
 class RadosSaveLog {
@@ -60,10 +63,16 @@ class RadosSaveLog {
     this->logfile = logfile_;
     log_active = !logfile.empty();
   }
+  RadosSaveLog() { log_active = false; }
+  void set_save_log_file(const std::string &logfile_) {
+    this->logfile = logfile_;
+    this->log_active = !logfile.empty();
+  }
   virtual ~RadosSaveLog(){};
   bool open();
   void append(const RadosSaveLogEntry &entry);
   bool close();
+  bool is_open() { return ofs.is_open(); }
 
  private:
   std::string logfile;

src/librmb/rados-storage-impl.cpp

@@ -201,6 +201,8 @@ int RadosStorageImpl::create_connection(const std::string &poolname) {
   if (err == 0) {
     io_ctx_created = true;
   }
+  // set the poolname
+  pool_name = poolname;
   return 0;
 }
 

src/librmb/rados-storage-impl.h

@@ -32,6 +32,8 @@ class RadosStorageImpl : public RadosStorage {
   int stat_mail(const std::string &oid, uint64_t *psize, time_t *pmtime);
   void set_namespace(const std::string &_nspace);
   std::string get_namespace() { return nspace; }
+  std::string get_pool_name() { return pool_name; }
+
   int get_max_write_size() { return max_write_size; }
   int get_max_write_size_bytes() { return max_write_size * 1024 * 1024; }
 
@@ -74,6 +76,7 @@ class RadosStorageImpl : public RadosStorage {
   std::string nspace;
   librados::IoCtx io_ctx;
   bool io_ctx_created;
+  std::string pool_name;
 
   static const char *CFG_OSD_MAX_WRITE_SIZE;
 };

src/librmb/rados-storage.h

@@ -32,6 +32,9 @@ class RadosStorage {
   virtual void set_namespace(const std::string &_nspace) = 0;
   /* get the object namespace */
   virtual std::string get_namespace() = 0;
+  /* get the pool name */
+  virtual std::string get_pool_name() = 0;
+
   /* get the max object size in mb */
   virtual int get_max_write_size() = 0;
   /* get the max object size in bytes */

src/storage-rbox/rbox-copy.cpp

@@ -185,34 +185,32 @@ static int rbox_mail_storage_try_copy(struct mail_save_context **_ctx, struct ma
     std::list<librmb::RadosMetadata> metadata_update;  // metadata which needs to be unique goes here
     std::string src_oid = rmail->mail_object->get_oid();
 
+    librmb::RadosStorage *rados_storage = !from_alt_storage ? r_storage->s : r_storage->alt;
+
     if (ctx->moving != TRUE) {
       setup_mail_object(ctx);
       std::string dest_oid = r_ctx->current_object->get_oid();
 
       set_mailbox_metadata(ctx, &metadata_update);
 
-      if (!from_alt_storage) {
-        ret_val = r_storage->s->copy(src_oid, ns_src.c_str(), dest_oid, ns_dest.c_str(), metadata_update);
-        if (ret_val < 0) {
-          i_error("copy mail failed: from namespace: %s to namespace %s: src_oid: %s, des_oid: %s, error_code: %d",
-                  ns_src.c_str(), ns_dest.c_str(), src_oid.c_str(), dest_oid.c_str(), ret_val);
-          FUNC_END_RET("ret == -1, rados_storage->copy failed");
-          r_storage->s->free_mail_object(r_ctx->current_object);
-          r_ctx->current_object = nullptr;
-          return -1;
-        }
-      } else {
-        ret_val = r_storage->alt->copy(src_oid, ns_src.c_str(), dest_oid, ns_dest.c_str(), metadata_update);
-        if (ret_val <0) {
-          i_error("copy mail failed: from namespace: %s to namespace %s: src_oid: %s, des_oid: %s, error_code: %d",
-                  ns_src.c_str(), ns_dest.c_str(), src_oid.c_str(), dest_oid.c_str(), ret_val);
-          FUNC_END_RET("ret == -1, rados_storage->copy failed");
-          r_storage->s->free_mail_object(r_ctx->current_object);
-          r_ctx->current_object = nullptr;
-          return -1;
-        }
+      ret_val = rados_storage->copy(src_oid, ns_src.c_str(), dest_oid, ns_dest.c_str(), metadata_update);
+      if (ret_val < 0) {
+        i_error(
+            "copy mail failed: from namespace: %s to namespace %s: src_oid: %s, des_oid: %s, error_code: %d, "
+            "storage_pool: %s",
+            ns_src.c_str(), ns_dest.c_str(), src_oid.c_str(), dest_oid.c_str(), ret_val,
+            rados_storage->get_pool_name().c_str());
+        FUNC_END_RET("ret == -1, rados_storage->copy failed");
+        rados_storage->free_mail_object(r_ctx->current_object);
+        r_ctx->current_object = nullptr;
+        return -1;
       }
+
       rbox_add_to_index(ctx);
+      if (r_storage->save_log->is_open()) {
+        r_storage->save_log->append(
+            librmb::RadosSaveLogEntry(dest_oid, ns_dest, rados_storage->get_pool_name(), "cpy"));
+      }
       i_debug("copy successfully finished: from src %s to oid = %s", src_oid.c_str(), dest_oid.c_str());
     }
     if (ctx->moving) {
@@ -221,24 +219,15 @@ static int rbox_mail_storage_try_copy(struct mail_save_context **_ctx, struct ma
       set_mailbox_metadata(ctx, &metadata_update);
 
       bool delete_source = true;
-      if (!from_alt_storage) {
-        ret_val =
-            r_storage->s->move(src_oid, ns_src.c_str(), dest_oid, ns_dest.c_str(), metadata_update, delete_source);
-        if (ret_val < 0) {
-          i_error("move mail failed: from namespace: %s to namespace %s: src_oid: %s, des_oid: %s, error_code : %d",
-                  ns_src.c_str(), ns_dest.c_str(), src_oid.c_str(), dest_oid.c_str(), ret_val);
-          FUNC_END_RET("ret == -1, rados_storage->move failed");
-          return -1;
-        }
-      } else {
-        ret_val =
-            r_storage->alt->move(src_oid, ns_src.c_str(), dest_oid, ns_dest.c_str(), metadata_update, delete_source);
-        if (ret_val < 0) {
-          i_error("move mail failed: from namespace: %s to namespace %s: src_oid: %s, des_oid: %s, error_code: %d",
-                  ns_src.c_str(), ns_dest.c_str(), src_oid.c_str(), dest_oid.c_str(), ret_val);
-          FUNC_END_RET("ret == -1, rados_storage->move failed");
-          return -1;
-        }
+      ret_val = rados_storage->move(src_oid, ns_src.c_str(), dest_oid, ns_dest.c_str(), metadata_update, delete_source);
+      if (ret_val < 0) {
+        i_error(
+            "move mail failed: from namespace: %s to namespace %s: src_oid: %s, des_oid: %s, error_code : %d, "
+            "pool_name: %s",
+            ns_src.c_str(), ns_dest.c_str(), src_oid.c_str(), dest_oid.c_str(), ret_val,
+            rados_storage->get_pool_name());
+        FUNC_END_RET("ret == -1, rados_storage->move failed");
+        return -1;
       }
 
       // set src as expunged
@@ -247,6 +236,9 @@ static int rbox_mail_storage_try_copy(struct mail_save_context **_ctx, struct ma
       array_append(&rmailbox->moved_items, &item, 1);
 
       rbox_move_index(ctx, mail);
+      if (r_storage->save_log->is_open()) {
+        r_storage->save_log->append(librmb::RadosSaveLogEntry(dest_oid, ns_dest, rados_storage->get_pool_name(), "mv"));
+      }
       i_debug("move successfully finished from %s (ns=%s) to %s (ns=%s)", src_oid.c_str(), ns_src.c_str(),
               src_oid.c_str(), ns_dest.c_str());
     }

src/storage-rbox/rbox-save.cpp

@@ -467,6 +467,10 @@ int rbox_save_finish(struct mail_save_context *_ctx) {
         i_error("saved mail: %s failed metadata_count %lu", r_ctx->current_object->get_oid().c_str(),
                 r_ctx->current_object->get_metadata()->size());
       }
+      if (r_storage->save_log->is_open()) {
+        r_storage->save_log->append(librmb::RadosSaveLogEntry(
+            r_ctx->current_object->get_oid(), r_storage->s->get_namespace(), r_storage->s->get_pool_name(), "save"));
+      }
     }
   }
 

src/storage-rbox/rbox-storage-struct.h

@@ -17,6 +17,7 @@
 #include "../librmb/rados-dovecot-ceph-cfg.h"
 #include "../librmb/rados-storage-impl.h"
 #include "../librmb/rados-metadata-storage-impl.h"
+#include "../librmb/rados-save-log.h"
 
 struct rbox_storage {
   struct mail_storage storage;
@@ -27,6 +28,7 @@ struct rbox_storage {
   librmb::RadosNamespaceManager *ns_mgr;
   librmb::RadosMetadataStorage *ms;
   librmb::RadosStorage *alt;
+  librmb::RadosSaveLog *save_log;
 };
 
 #endif  // SRC_STORAGE_RBOX_RBOX_STORAGE_STRUCT_H_

src/storage-rbox/rbox-storage.cpp

@@ -72,6 +72,10 @@ struct mail_storage *rbox_storage_alloc(void) {
   storage->ns_mgr = new librmb::RadosNamespaceManager(storage->config);
   storage->ms = new librmb::RadosMetadataStorageImpl();
   storage->alt = new librmb::RadosStorageImpl(storage->cluster);
+
+  // logfile is set when 90-plugin.conf param rados_save_cfg is evaluated.
+  storage->save_log = new librmb::RadosSaveLog();
+
   FUNC_END();
   return &storage->storage;
 }
@@ -192,7 +196,13 @@ void rbox_storage_destroy(struct mail_storage *_storage) {
     delete storage->ms;
     storage->ms = nullptr;
   }
-
+  if (storage->save_log != nullptr) {
+    if (!storage->save_log->close()) {
+      i_warning("unable to close save log file");
+    }
+    delete storage->save_log;
+    storage->save_log = nullptr;
+  }
   index_storage_destroy(_storage);
 
   FUNC_END();
@@ -341,6 +351,10 @@ int read_plugin_configuration(struct mailbox *box) {
       storage->config->update_metadata(setting, mail_user_plugin_getenv(mbox->storage->storage.user, setting.c_str()));
     }
     storage->config->set_config_valid(true);
+    storage->save_log->set_save_log_file(storage->config->get_rados_save_log_file());
+    if (!storage->save_log->open()) {
+      i_warning("unable to open the rados save log file ", storage->config->get_rados_save_log_file().c_str());
+    }
   }
   FUNC_END();
   return 0;

src/storage-rbox/rbox-storage.hpp

@@ -28,6 +28,7 @@
 #include "../librmb/rados-namespace-manager.h"
 #include "../librmb/rados-dovecot-ceph-cfg.h"
 #include "../librmb/rados-metadata-storage-impl.h"
+#include "../librmb/rados-save-log.h"
 
 struct rbox_storage {
   struct mail_storage storage;
@@ -38,6 +39,7 @@ struct rbox_storage {
   librmb::RadosNamespaceManager *ns_mgr;
   librmb::RadosMetadataStorage *ms;
   librmb::RadosStorage *alt;
+  librmb::RadosSaveLog *save_log;
 };
 
 #endif

src/tests/librmb/test_librmb_utils.cpp

@@ -130,7 +130,7 @@ TEST(librmb, append_to_new_log_file) {
   std::string test_file_name = "test.log";
   librmb::RadosSaveLog log_file(test_file_name);
   EXPECT_EQ(true, log_file.open());
-  log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage"));
+  log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage", "save"));
   EXPECT_EQ(true, log_file.close());
 
   int line_count = 0;
@@ -146,6 +146,7 @@ TEST(librmb, append_to_new_log_file) {
     EXPECT_EQ(entry.oid, "abc");
     EXPECT_EQ(entry.ns, "ns_1");
     EXPECT_EQ(entry.pool, "mail_storage");
+    EXPECT_EQ(entry.op, "save");
     line_count++;
   }
   EXPECT_EQ(1, line_count);
@@ -157,11 +158,11 @@ TEST(librmb, append_to_existing_file_log_file) {
   std::string test_file_name = "test.log";
   librmb::RadosSaveLog log_file(test_file_name);
   EXPECT_EQ(true, log_file.open());
-  log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage"));
+  log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage", "save"));
   EXPECT_EQ(true, log_file.close());
 
   EXPECT_EQ(true, log_file.open());
-  log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage"));
+  log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage", "save"));
   EXPECT_EQ(true, log_file.close());
 
   int line_count = 0;
@@ -177,6 +178,7 @@ TEST(librmb, append_to_existing_file_log_file) {
     EXPECT_EQ(entry.oid, "abc");
     EXPECT_EQ(entry.ns, "ns_1");
     EXPECT_EQ(entry.pool, "mail_storage");
+    EXPECT_EQ(entry.op, "save");
     line_count++;
   }
   EXPECT_EQ(2, line_count);
@@ -189,7 +191,7 @@ void *write_to_save_file(void *threadid) {
   librmb::RadosSaveLog log_file(test_file_name);
   EXPECT_EQ(true, log_file.open());
   for (int i = 0; i < 5; i++) {
-    log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage"));
+    log_file.append(librmb::RadosSaveLogEntry("abc", "ns_1", "mail_storage", "save"));
   }
   EXPECT_EQ(true, log_file.close());
   std::cout << " exiting thread " << threadid << std::endl;
@@ -237,6 +239,7 @@ TEST(librmb, append_to_existing_file_multi_threading) {
     EXPECT_EQ(entry.oid, "abc");
     EXPECT_EQ(entry.ns, "ns_1");
     EXPECT_EQ(entry.pool, "mail_storage");
+    EXPECT_EQ(entry.op, "save");
     line_count++;
   }
   EXPECT_EQ(25, line_count);

src/tests/mocks/mock_test.h

@@ -34,6 +34,8 @@ class RadosStorageMock : public RadosStorage {
   MOCK_METHOD3(stat_mail, int(const std::string &oid, uint64_t *psize, time_t *pmtime));
   MOCK_METHOD1(set_namespace, void(const std::string &nspace));
   MOCK_METHOD0(get_namespace, std::string());
+  MOCK_METHOD0(get_pool_name, std::string());
+
   MOCK_METHOD0(get_max_write_size, int());
   MOCK_METHOD0(get_max_write_size_bytes, int());
 
@@ -133,6 +135,7 @@ class RadosDovecotCephCfgMock : public RadosDovecotCephCfg {
   // dovecot configuration
   MOCK_METHOD0(get_rados_cluster_name, const std::string &());
   MOCK_METHOD0(get_rados_username, const std::string &());
+  MOCK_METHOD0(get_rados_save_log_file, const std::string &());
 
   // dovecot configuration
   MOCK_METHOD1(is_mail_attribute, bool(enum librmb::rbox_metadata_key key));