Store folders realtime sync status. Done #107

sync/README.md

@@ -76,7 +76,7 @@ larger packet size to be sent in a query. To do this, add the following line to
 your SQL config file:
 
     [mysqld]
-    max_allowed_pack = 512M
+    max_allowed_packet = 512M
 
 You don't have to use 512MB as your packet size, but anything 16MB or higher is
 advised.

sync/config/default.ini

@@ -13,6 +13,7 @@ db[sleep_minutes] = 1
 sync[wait_seconds] = 10
 sync[sleep_minutes] = 15
 timezone = "America/New_York"
+ignored_folders[] = "[Gmail]"
 
 [log]
 
@@ -36,6 +37,7 @@ hostname = "localhost"
 
 services[] = "GMail"
 services[] = "Yahoo"
+services[] = "Office365"
 services[] = "Other"
 gmail[domain] = "gmail.com"
 gmail[host] = "imap.gmail.com"
@@ -47,8 +49,12 @@ yahoo[host] = "imap.mail.yahoo.com"
 yahoo[port] = 993
 yahoo[smtp_host] = "imap.mail.yahoo.com"
 yahoo[smtp_port] = "smtp.mail.yahoo.com"
+office365[host] = "outlook.office365.com"
+office365[port] = 993
+office365[smtp_host] = "smtp.office365.com"
+office365[smtp_port] = 587
 other[port] = 993
-other[smtp_port] = 993
+other[smtp_port] = 587
 attachments[path] = "attachments"
 
 [daemon]

sync/db/003_folders.sql

@@ -3,6 +3,7 @@ CREATE TABLE IF NOT EXISTS `folders` (
   `account_id` int(10) unsigned NULL,
   `name` varchar(150) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
   `count` int(10) unsigned DEFAULT '0',
+  `status` ENUM('not_synced','syncing', 'syncing_need_resync','synced','synced') NOT NULL DEFAULT 'not_synced',
   `synced` int(10) unsigned DEFAULT '0',
   `uid_validity` int(10) unsigned DEFAULT '0',
   `deleted` tinyint(1) unsigned DEFAULT '0',

sync/src/Console/SyncConsole.php

@@ -21,6 +21,7 @@ class SyncConsole extends Console
     public $sleep;
     public $create;
     public $folder;
+    public $email;
     public $daemon;
     public $actions;
     public $verbose;
@@ -81,6 +82,11 @@ protected function setupArgs()
                 'longPrefix' => 'folder',
                 'description' => 'Sync the selected folder'
             ],
+            'email' => [
+                'prefix' => 'm',
+                'longPrefix' => 'email',
+                'description' => 'Sync the selected email account (passing email)'
+            ],
             'help' => [
                 'prefix' => 'h',
                 'longPrefix' => 'help',
@@ -163,6 +169,7 @@ protected function parseArgs()
         $this->diagnostics = $this->cli->arguments->get('diagnostics');
         $this->interactive = $this->cli->arguments->get('interactive');
         $this->databaseExists = $this->cli->arguments->get('exists');
+        $this->email = $this->cli->arguments->get('email');
 
         // Some flags also enable interactive mode
         if (true === $this->sleep

sync/src/Enum/FolderSyncStatus.php

@@ -0,0 +1,49 @@
+<?php
+
+namespace App\Enum;
+
+class FolderSyncStatus {
+
+    const __default = self::NotSynced;
+
+    /**
+     * Initial state, when folder just added to database and syncing process has never processed.
+     */
+    const NotSynced = 'not_synced';
+
+    /**
+     * This folder is syncing now.
+     */
+    const Syncing = 'syncing';
+
+    /**
+     * This folder is syncing now, and we received one or more request for resync after syncing process started.
+     * Example: syncing process started and user delete / change flag of message, which already synced.
+     * We can not miss this event and resyncing needed.
+     */
+    const SyncingNeedResync = 'syncing_need_resync';
+
+    /**
+     * Sync process completed and there is no new events received from imap server ("mail" / "update" / "purge").
+     */
+    const Synced = 'synced';
+
+    /**
+     * There was some error, during sync process.
+     */
+    const Error = 'error';
+
+    protected static $choices = [
+        self::NotSynced  => 'Not synced',
+        self::Syncing   => 'Syncing',
+        self::SyncingNeedResync  => 'Syncing but need resync',
+        self::Synced => 'Synced',
+        self::Error => 'Error'
+    ];
+
+
+    public static function getValues(): array
+    {
+        return \array_keys(static::$choices);
+    }
+}

sync/src/Model/Folder.php

@@ -3,6 +3,7 @@
 namespace App\Model;
 
 use Fn;
+use App\Enum\FolderSyncStatus;
 use PDO;
 use DateTime;
 use App\Model;
@@ -339,4 +340,38 @@ public function getSentByAccount(int $accountId)
 
         throw new NotFoundException('sent mail folder');
     }
+
+    /**
+     * @param string $status
+     * @return int
+     * @throws DatabaseUpdateException
+     */
+    public function updateStatus(string $status)
+    {
+        if (!in_array($status, FolderSyncStatus::getValues())) {
+            throw new \LogicException("Unsupported folder sync status '{$status}'");
+        }
+
+        $updated = $this->db()
+            ->update(['status' => $status])
+            ->table('folders')
+            ->where('id', '=', $this->getId())
+            ->execute();
+        $this->errorHandle($updated);
+        return $updated;
+    }
+
+    /**
+     * @return string
+     */
+    public function getStatus()
+    {
+        $row = $this->db()
+            ->select(['status'])
+            ->from('folders')
+            ->where('id', '=', $this->getId())
+            ->execute()
+            ->fetch();
+        return $row ? $row['status'] : FolderSyncStatus::__default;
+    }
 }

sync/src/Model/Message.php

@@ -1051,19 +1051,4 @@ private function formatAttachments(array $attachments)
 
         return json_encode($formatted, JSON_UNESCAPED_SLASHES);
     }
-
-    /**
-     * @param bool | int $updated Response from update operation
-     *
-     * @throws DatabaseUpdateException
-     */
-    private function errorHandle($updated)
-    {
-        if (! Belt::isNumber($updated)) {
-            throw new DatabaseUpdateException(
-                MESSAGE,
-                $this->getError()
-            );
-        }
-    }
 }

sync/src/Sync.php

@@ -7,6 +7,7 @@
 namespace App;
 
 use Fn;
+use App\Enum\FolderSyncStatus;
 use DateTime;
 use Exception;
 use PDOException;
@@ -57,6 +58,8 @@ class Sync
     private $maxRetries = 5;
     private $retriesFolders;
     private $retriesMessages;
+    private $email;
+    private $ignoredFolders;
 
     // Config
     const READY_THRESHOLD = 60;
@@ -99,15 +102,37 @@ public function __construct(Container $di = null)
             $this->quick = $di['console']->quick;
             $this->sleep = $di['console']->sleep;
             $this->folder = $di['console']->folder;
+            $this->email = $di['console']->email;
             $this->daemon = $di['console']->daemon;
             $this->actions = $di['console']->actions;
             $this->threading = $di['console']->threading;
             $this->interactive = $di['console']->interactive;
+            if (array_key_exists('ignored_folders', $di['config']['app'])) {
+                $this->ignoredFolders = $di['config']['app']['ignored_folders'];
+            } else {
+                $this->ignoredFolders = [];
+            }
         }
 
         $this->initGc();
     }
 
+    /**
+     * @return mixed
+     */
+    public function getEmail()
+    {
+        return $this->email;
+    }
+
+    /**
+     * @param mixed $email
+     */
+    public function setEmail($email)
+    {
+        $this->email = $email;
+    }
+
     /**
      * @param CLImate $cli
      */
@@ -132,6 +157,22 @@ public function setConfig(array $config)
         $this->config = $config;
     }
 
+    /**
+     * @return array
+     */
+    public function getIgnoredFolders(): array
+    {
+        return $this->ignoredFolders;
+    }
+
+    /**
+     * @param array $ignoredFolders
+     */
+    public function setIgnoredFolders(array $ignoredFolders)
+    {
+        $this->ignoredFolders = $ignoredFolders;
+    }
+
     /**
      * Runs sync forever. This is a while loop that runs a sync
      * for all accounts, then sleeps for a designated period of
@@ -141,6 +182,7 @@ public function loop()
     {
         $wakeUnix = 0;
         $sleepMinutes = $this->config['app']['sync']['sleep_minutes'];
+        $account = ((new AccountModel)->getByEmail($this->email)) ?: null;
 
         while (true) {
             $this->gc();
@@ -155,7 +197,7 @@ public function loop()
                 // Run action sync every minute
                 if ($this->isReadyToRun()) {
                     $this->setAsleep(false);
-                    $this->run(null, [self::OPT_ONLY_SYNC_ACTIONS => true]);
+                    $this->run($account, [self::OPT_ONLY_SYNC_ACTIONS => true]);
                     $this->setAsleep(true);
                 }
 
@@ -165,7 +207,7 @@ public function loop()
 
             $this->setAsleep(false);
 
-            if (! $this->run()) {
+            if (! $this->run($account)) {
                 throw new TerminateException('Sync was prevented from running');
             }
 
@@ -568,7 +610,8 @@ private function syncFolders(AccountModel $account)
                 $this->log,
                 $this->cli,
                 $this->emitter,
-                $this->interactive
+                $this->interactive,
+                $this->ignoredFolders
             );
             $folderList = $this->mailbox->getFolders();
             $savedFolders = (new FolderModel)->getByAccount($account->getId());
@@ -612,12 +655,19 @@ private function syncMessages(AccountModel $account, array $folders, array $opti
             $this->log->debug('Syncing messages in each folder');
         }
 
+        /** @var FolderModel $folder */
         foreach ($folders as $folder) {
             $this->retriesMessages[$account->email] = 1;
             $this->stats->setActiveFolder($folder->name);
 
             try {
+                $folder->updateStatus(FolderSyncStatus::Syncing);
                 $this->syncFolderMessages($account, $folder, $options);
+                // We received new 'mail' / 'update' / 'purge' event from IMAP server, during sync process.
+                if ($folder->getStatus() == FolderSyncStatus::SyncingNeedResync) {
+                    $this->syncFolderMessages($account, $folder, $options);
+                }
+                $folder->updateStatus(FolderSyncStatus::Synced);
             } catch (MessagesSyncException $e) {
                 $this->log->error($e->getMessage());
             }
@@ -707,12 +757,16 @@ private function syncFolderMessages(
             $messageSync->run($account, $folder, $selectStats, $options);
             $this->checkForHalt();
         } catch (PDOException $e) {
+            $folder->updateStatus(FolderSyncStatus::Error);
             throw $e;
         } catch (StopException $e) {
+            $folder->updateStatus(FolderSyncStatus::Error);
             throw $e;
         } catch (TerminateException $e) {
+            $folder->updateStatus(FolderSyncStatus::Error);
             throw $e;
         } catch (Exception $e) {
+            $folder->updateStatus(FolderSyncStatus::Error);
             $this->stats->unsetActiveFolder();
             $this->log->error(substr($e->getMessage(), 0, 500));
             $this->checkForClosedConnection($e);
@@ -828,4 +882,6 @@ private function checkForClosedConnection(Exception $e)
             throw new StopException;
         }
     }
+
+
 }