117 track decryption status of files, messages, replies

alembic/versions/2f363b3d680e_init.py

@@ -52,6 +52,7 @@ def upgrade():
         sa.Column('is_downloaded', sa.Boolean(name='is_downloaded'), server_default='0',
                   nullable=False),
         sa.Column('is_read', sa.Boolean(name='is_read'), server_default='0', nullable=False),
+        sa.Column('is_decrypted', sa.Boolean(name='is_decrypted'), nullable=True),
         sa.Column('source_id', sa.Integer(), nullable=True),
         sa.ForeignKeyConstraint(['source_id'], ['sources.id'],
                                 name=op.f('fk_files_source_id_sources')),
@@ -69,6 +70,7 @@ def upgrade():
         sa.Column('is_downloaded', sa.Boolean(name='is_downloaded'), server_default='0',
                   nullable=False),
         sa.Column('is_read', sa.Boolean(name='is_read'), server_default='0', nullable=False),
+        sa.Column('is_decrypted', sa.Boolean(name='is_decrypted'), nullable=True),
         sa.Column('content', sa.Text(), nullable=True),
         sa.Column('source_id', sa.Integer(), nullable=True),
         sa.ForeignKeyConstraint(['source_id'], ['sources.id'],
@@ -88,6 +90,7 @@ def upgrade():
         sa.Column('filename', sa.String(length=255), nullable=False),
         sa.Column('size', sa.Integer(), nullable=True),
         sa.Column('is_downloaded', sa.Boolean(name='is_downloaded'), nullable=True),
+        sa.Column('is_decrypted', sa.Boolean(name='is_decrypted'), nullable=True),
         sa.ForeignKeyConstraint(['journalist_id'], ['users.id'],
                                 name=op.f('fk_replies_journalist_id_users')),
         sa.ForeignKeyConstraint(['source_id'], ['sources.id'],

securedrop_client/db.py

@@ -78,6 +78,9 @@ class Message(Base):
     # This is whether the submission has been downloaded in the local database.
     is_downloaded = Column(Boolean(name='is_downloaded'), nullable=False, server_default="0")
 
+    # This tracks if the file had been successfully decrypted after download.
+    is_decrypted = Column(Boolean(name='is_decrypted'), nullable=True)
+
     # This reflects read status stored on the server.
     is_read = Column(Boolean(name='is_read'), nullable=False, server_default="0")
 
@@ -110,6 +113,9 @@ class File(Base):
     # This is whether the submission has been downloaded in the local database.
     is_downloaded = Column(Boolean(name='is_downloaded'), nullable=False, server_default="0")
 
+    # This tracks if the file had been successfully decrypted after download.
+    is_decrypted = Column(Boolean(name='is_decrypted'), nullable=True)
+
     # This reflects read status stored on the server.
     is_read = Column(Boolean(name='is_read'), nullable=False, server_default="0")
 
@@ -144,6 +150,9 @@ class Reply(Base):
     is_downloaded = Column(Boolean(name='is_downloaded'),
                            default=False)
 
+    # This tracks if the file had been successfully decrypted after download.
+    is_decrypted = Column(Boolean(name='is_decrypted'), nullable=True)
+
     def __repr__(self):
         return '<Reply {}>'.format(self.filename)
 

securedrop_client/logic.py

@@ -612,31 +612,30 @@ def on_file_downloaded(self, result, current_object):
         file_uuid = current_object.uuid
         server_filename = current_object.filename
         if isinstance(result, tuple):  # The file properly downloaded.
-            sha256sum, filename = result
+            _, filename = result
             # The filename contains the location where the file has been
             # stored. On non-Qubes OSes, this will be the data directory.
             # On Qubes OS, this will a ~/QubesIncoming directory. In case
             # we are on Qubes, we should move the file to the data directory
             # and name it the same as the server (e.g. spotless-tater-msg.gpg).
             filepath_in_datadir = os.path.join(self.data_dir, server_filename)
             shutil.move(filename, filepath_in_datadir)
+            storage.mark_file_as_downloaded(file_uuid, self.session)
 
             try:
                 # Attempt to decrypt the file.
                 self.gpg.decrypt_submission_or_reply(
                     filepath_in_datadir, server_filename, is_doc=True)
+                storage.set_object_decryption_status(file_uuid, self.session, db.File, True)
             except CryptoError as e:
                 logger.debug('Failed to decrypt file {}: {}'.format(server_filename, e))
-
-                self.set_status("Failed to download and decrypt file, "
-                                "please try again.")
+                storage.set_object_decryption_status(file_uuid, self.session, db.File, False)
+                self.set_status("Failed to decrypt file, "
+                                "please try again or talk to your administrator.")
                 # TODO: We should save the downloaded content, and just
                 # try to decrypt again if there was a failure.
                 return  # If we failed we should stop here.
 
-            # Now that download and decrypt are done, mark the file as such.
-            storage.mark_file_as_downloaded(file_uuid, self.session)
-
             self.set_status('Finished downloading {}'.format(current_object.filename))
         else:  # The file did not download properly.
             logger.debug('Failed to download file {}'.format(server_filename))

securedrop_client/message_sync.py

@@ -25,8 +25,8 @@
 
 from PyQt5.QtCore import QObject, pyqtSignal
 from securedrop_client import storage
-from securedrop_client.crypto import GpgHelper
-from securedrop_client.db import make_engine, Message
+from securedrop_client.crypto import GpgHelper, CryptoError
+from securedrop_client.db import make_engine
 from securedrop_client.storage import get_data
 from sqlalchemy.orm import sessionmaker
 
@@ -48,11 +48,18 @@ def __init__(self, api, home, is_qubes):
         self.gpg = GpgHelper(home, is_qubes)
 
     def fetch_the_thing(self, item, msg, download_fn, update_fn):
-        shasum, filepath = download_fn(item)
-        self.gpg.decrypt_submission_or_reply(filepath, msg.filename, False)
+        _, filepath = download_fn(item)
         update_fn(msg.uuid, self.session)
         logger.info("Stored message or reply at {}".format(msg.filename))
 
+        try:
+            self.gpg.decrypt_submission_or_reply(filepath, msg.filename, False)
+            storage.set_object_decryption_status(msg.uuid, self.session, type(msg), True)
+            logger.info("Message or reply decrypted: {}".format(msg.filename))
+        except CryptoError:
+            storage.set_object_decryption_status(msg.uuid, self.session, type(msg), False)
+            logger.info("Message or reply failed to decrypt: {}".format(msg.filename))
+
 
 class MessageSync(APISyncObject):
     """
@@ -71,7 +78,6 @@ def __init__(self, api, home, is_qubes):
     def run(self, loop=True):
         while True:
             submissions = storage.find_new_messages(self.session)
-            submissions.extend(storage.find_new_files(self.session))
 
             for db_submission in submissions:
                 try:
@@ -82,16 +88,11 @@ def run(self, loop=True):
                     # Need to set filename on non-Qubes platforms
                     sdk_submission.filename = db_submission.filename
 
-                    if isinstance(db_submission, Message):
-                        callback = storage.mark_message_as_downloaded
-                    else:
-                        callback = storage.mark_file_as_downloaded
-
                     if self.api:
                         self.fetch_the_thing(sdk_submission,
                                              db_submission,
                                              self.api.download_submission,
-                                             callback)
+                                             storage.mark_message_as_downloaded)
                         self.message_downloaded.emit(db_submission.uuid,
                                                      get_data(self.home, db_submission.filename))
                 except Exception:

securedrop_client/storage.py

@@ -96,8 +96,8 @@ def update_local_storage(session, remote_sources, remote_submissions,
     local_messages = get_local_messages(session)
     local_replies = get_local_replies(session)
 
-    remote_messages = [x for x in remote_submissions if x.filename.endswith('.msg.gpg')]
-    remote_files = [x for x in remote_submissions if not x.filename.endswith('.msg.gpg')]
+    remote_messages = [x for x in remote_submissions if x.filename.endswith('msg.gpg')]
+    remote_files = [x for x in remote_submissions if not x.filename.endswith('msg.gpg')]
 
     update_sources(remote_sources, local_sources, session, data_dir)
     update_files(remote_files, local_files, session, data_dir)
@@ -325,6 +325,14 @@ def mark_message_as_downloaded(uuid, session):
     session.commit()
 
 
+def set_object_decryption_status(uuid, session, model, is_successful: bool):
+    """Mark object as decrypted or not in the database."""
+    db_object = session.query(model).filter_by(uuid=uuid).one_or_none()
+    db_object.is_decrypted = is_successful
+    session.add(db_object)
+    session.commit()
+
+
 def mark_reply_as_downloaded(uuid, session):
     """
     Mark reply as downloaded in the database.

tests/test_logic.py

@@ -988,6 +988,8 @@ def test_Client_on_file_downloaded_success(homedir, config, mocker):
     mock_gpg.call_count == 1
     mock_storage.mark_file_as_downloaded.assert_called_once_with(
         test_object_uuid, mock_session)
+    mock_storage.set_object_decryption_status.assert_called_once_with(
+        submission_db_object.uuid, mock_session, db.File, True)
 
 
 def test_Client_on_file_downloaded_api_failure(homedir, config, mocker):
@@ -1030,12 +1032,15 @@ def test_Client_on_file_downloaded_decrypt_failure(homedir, config, mocker):
     submission_db_object.filename = 'filename'
     mock_gpg = mocker.patch.object(cl.gpg, 'decrypt_submission_or_reply',
                                    side_effect=CryptoError())
+    mock_storage = mocker.patch('securedrop_client.logic.storage')
     mocker.patch('shutil.move')
 
     cl.on_file_downloaded(result_data, current_object=submission_db_object)
     mock_gpg.call_count == 1
     cl.set_status.assert_called_once_with(
-        "Failed to download and decrypt file, please try again.")
+        "Failed to decrypt file, please try again or talk to your administrator.")
+    mock_storage.set_object_decryption_status.assert_called_once_with(
+        submission_db_object.uuid, mock_session, db.File, False)
 
 
 def test_Client_on_file_download_user_not_signed_in(homedir, config, mocker):

tests/test_message_sync.py

@@ -30,16 +30,18 @@ def test_MessageSync_init(mocker):
 
 
 def test_MessageSync_run_success(mocker, source):
-    submission1 = File(source=source['source'], uuid='uuid1', filename='filename',
-                       download_url='http://test.net')
-    submission2 = Message(source=source['source'], uuid='uuid2', filename='filename',
-                          download_url='http://test.net')
+    """Test when a message successfully downloads and decrypts."""
+
+    submission = Message(source=source['source'], uuid='uuid2', filename='filename',
+                         download_url='http://test.net')
 
     # mock the fetching of submissions
-    mocker.patch('securedrop_client.storage.find_new_messages', return_value=[submission1])
-    mocker.patch('securedrop_client.storage.find_new_files', return_value=[submission2])
-    mocker.patch('securedrop_client.message_sync.storage.mark_file_as_downloaded')
-    mocker.patch('securedrop_client.message_sync.storage.mark_message_as_downloaded')
+    mocker.patch('securedrop_client.storage.find_new_messages', return_value=[submission])
+    mock_download_status = mocker.patch(
+        'securedrop_client.message_sync.storage.mark_message_as_downloaded')
+    mock_decryption_status = mocker.patch(
+        'securedrop_client.message_sync.storage.set_object_decryption_status')
+
     # don't create the signal
     mocker.patch('securedrop_client.message_sync.pyqtSignal')
     # mock the GpgHelper creation since we don't have directories/keys setup
@@ -61,6 +63,50 @@ def test_MessageSync_run_success(mocker, source):
     ms.run(False)
 
     assert mock_emit.called
+    assert mock_decryption_status.called_with(submission.uuid, api.session, type(submission), False)
+    assert mock_download_status.called_with(submission.uuid, api.mock_session)
+
+
+def test_MessageSync_run_decryption_error(mocker, source):
+    """Test when a message successfully downloads, but does not successfully decrypt."""
+
+    submission = File(source=source['source'], uuid='uuid1', filename='filename',
+                      download_url='http://test.net')
+
+    # mock the fetching of submissions
+    mocker.patch('securedrop_client.storage.find_new_messages', return_value=[submission])
+    mock_download_status = mocker.patch(
+        'securedrop_client.message_sync.storage.mark_message_as_downloaded')
+    mock_decryption_status = mocker.patch(
+        'securedrop_client.message_sync.storage.set_object_decryption_status')
+
+    # don't create the signal
+    mocker.patch('securedrop_client.message_sync.pyqtSignal')
+    # mock the GpgHelper creation since we don't have directories/keys setup
+    mocker.patch('securedrop_client.message_sync.GpgHelper')
+
+    api = mocker.MagicMock()
+    api.session = mocker.MagicMock()
+    home = "/home/user/.sd"
+    is_qubes = True
+
+    ms = MessageSync(api, home, is_qubes)
+    mocker.patch.object(ms.gpg, 'decrypt_submission_or_reply', side_effect=CryptoError)
+
+    ms.api.download_submission = mocker.MagicMock(return_value=(1234, "/home/user/downloads/foo"))
+
+    mock_message_downloaded = mocker.Mock()
+    mock_emit = mocker.Mock()
+    mock_message_downloaded.emit = mock_emit
+    mocker.patch.object(ms, 'message_downloaded', new=mock_message_downloaded)
+
+    # check that it runs without raising exceptions
+    ms.run(False)
+
+    assert mock_download_status.called_with(submission.uuid, api.mock_session)
+    assert mock_decryption_status.called_with(submission.uuid, api.session,
+                                              type(submission), False)
+    assert mock_emit.called
 
 
 def test_MessageSync_exception(homedir, config, mocker):
@@ -128,6 +174,7 @@ def test_ReplySync_run_success(mocker):
     mocker.patch('securedrop_client.message_sync.pyqtSignal')
     # mock the handling of the replies
     mocker.patch('securedrop_client.message_sync.storage.mark_reply_as_downloaded')
+    mocker.patch('securedrop_client.message_sync.storage.set_object_decryption_status')
     mocker.patch('securedrop_client.message_sync.GpgHelper')
 
     api = mocker.MagicMock()

tests/test_storage.py

@@ -10,7 +10,7 @@
     get_remote_data, update_local_storage, update_sources, update_files, update_replies, \
     find_or_create_user, find_new_messages, find_new_replies, mark_file_as_downloaded, \
     mark_reply_as_downloaded, delete_single_submission_or_reply_on_disk, get_data, rename_file, \
-    get_local_files, find_new_files, mark_message_as_downloaded
+    get_local_files, find_new_files, mark_message_as_downloaded, set_object_decryption_status
 from securedrop_client import db
 from sdclientapi import Source, Submission, Reply
 
@@ -664,6 +664,36 @@ def test_find_new_replies(mocker):
     assert replies[0].is_downloaded is False
 
 
+def test_set_object_decryption_status_null_to_false(mocker):
+    mock_session = mocker.MagicMock()
+    mock_file = mocker.MagicMock()
+    mock_file.is_decrypted is None
+    mock_session.query().filter_by().one_or_none.return_value = mock_file
+
+    decryption_status = False
+    set_object_decryption_status('uuid', mock_session, db.File, decryption_status)
+
+    assert mock_file.is_decrypted is False
+
+    mock_session.add.assert_called_once_with(mock_file)
+    mock_session.commit.assert_called_once_with()
+
+
+def test_set_object_decryption_status_false_to_true(mocker):
+    mock_session = mocker.MagicMock()
+    mock_file = mocker.MagicMock()
+    mock_file.is_decrypted is False
+    mock_session.query().filter_by().one_or_none.return_value = mock_file
+
+    decryption_status = True
+    set_object_decryption_status('uuid', mock_session, db.File, decryption_status)
+
+    assert mock_file.is_decrypted is True
+
+    mock_session.add.assert_called_once_with(mock_file)
+    mock_session.commit.assert_called_once_with()
+
+
 def test_mark_file_as_downloaded(mocker):
     mock_session = mocker.MagicMock()
     mock_submission = mocker.MagicMock()