Add AWS S3 sync #2815

alexchee · 2020-03-20T01:53:12Z

Adds syncing with AWS bucket, requires read/write access to the whole bucket. DO NOT share bucket with any thing else, it will erase them.

I am able to get it to sync with my s3 bucket over the electron app with resources and E2EE enabled, but tests are failing. I'm making a PR it here in case someone can help me figure out what is causing the fails. Also I haven't had much experience with React Native and can use someone to help me test that.

Setup

signup for AWS account, if you need one
create an S3 bucket, make sure to default to Block all public access if you don't want your notes available to public: https://s3.console.aws.amazon.com/
You should create an AWS User just for joplin to keep: https://console.aws.amazon.com/iam/home?region=us-east-1#/users
For extra access control, make sure the user can only read and write to s3 bucket. Here's an example Policy, if the bucket is called my-joplin-bucket:

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "VisualEditor0",
            "Effect": "Allow",
            "Action": [
                "s3:PutAnalyticsConfiguration",
                "s3:GetObjectVersionTagging",
                "s3:CreateBucket",
                "s3:ReplicateObject",
                "s3:GetObjectAcl",
                "s3:GetBucketObjectLockConfiguration",
                "s3:DeleteBucketWebsite",
                "s3:PutLifecycleConfiguration",
                "s3:GetObjectVersionAcl",
                "s3:PutBucketAcl",
                "s3:PutObjectTagging",
                "s3:HeadBucket",
                "s3:DeleteObject",
                "s3:DeleteObjectTagging",
                "s3:GetBucketPolicyStatus",
                "s3:GetObjectRetention",
                "s3:GetBucketWebsite",
                "s3:PutReplicationConfiguration",
                "s3:DeleteObjectVersionTagging",
                "s3:PutObjectLegalHold",
                "s3:GetObjectLegalHold",
                "s3:GetBucketNotification",
                "s3:PutBucketCORS",
                "s3:DeleteBucketPolicy",
                "s3:GetReplicationConfiguration",
                "s3:ListMultipartUploadParts",
                "s3:PutObject",
                "s3:GetObject",
                "s3:PutBucketNotification",
                "s3:PutBucketLogging",
                "s3:PutObjectVersionAcl",
                "s3:GetAnalyticsConfiguration",
                "s3:PutBucketObjectLockConfiguration",
                "s3:GetObjectVersionForReplication",
                "s3:GetLifecycleConfiguration",
                "s3:GetInventoryConfiguration",
                "s3:GetBucketTagging",
                "s3:PutAccelerateConfiguration",
                "s3:DeleteObjectVersion",
                "s3:GetBucketLogging",
                "s3:ListBucketVersions",
                "s3:ReplicateTags",
                "s3:RestoreObject",
                "s3:ListBucket",
                "s3:GetAccelerateConfiguration",
                "s3:GetBucketPolicy",
                "s3:PutEncryptionConfiguration",
                "s3:GetEncryptionConfiguration",
                "s3:GetObjectVersionTorrent",
                "s3:AbortMultipartUpload",
                "s3:PutBucketTagging",
                "s3:GetBucketRequestPayment",
                "s3:GetObjectTagging",
                "s3:GetMetricsConfiguration",
                "s3:DeleteBucket",
                "s3:PutBucketVersioning",
                "s3:PutObjectAcl",
                "s3:GetBucketPublicAccessBlock",
                "s3:ListBucketMultipartUploads",
                "s3:ListAccessPoints",
                "s3:PutMetricsConfiguration",
                "s3:PutObjectVersionTagging",
                "s3:GetBucketVersioning",
                "s3:GetBucketAcl",
                "s3:PutInventoryConfiguration",
                "s3:GetObjectTorrent",
                "s3:ObjectOwnerOverrideToBucketOwner",
                "s3:PutBucketWebsite",
                "s3:PutBucketRequestPayment",
                "s3:PutObjectRetention",
                "s3:GetBucketCORS",
                "s3:PutBucketPolicy",
                "s3:GetBucketLocation",
                "s3:ReplicateDelete",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::my-joplin-bucket",
                "arn:aws:s3:::my-joplin-bucket/*"
            ]
        }
    ]
}

Get Create access key under User's security credentials, and save the ID and Secret.
Open Joplin, Settings, Synchronization
Select target AWS S3
Enter the AWS bucket name, user's key ID, and secret

Running

cd ElectronClient
npm start

Testing

Create CliClient/tests/support/amazon-s3-auth.json with your AWS credentials. Should use a different AWS bucket than your dev one to prevent it deleting all your dev content.

{"bucket": "my-joplin-bucket", "accessKeyId": "AKIAZ1234567890", "secretAccessKey": "SECRET"}

InCliClient/tests/test-utils.js, comment out

// const syncTargetId_ = SyncTargetRegistry.nameToId('memory');

uncomment

const syncTargetId_ = SyncTargetRegistry.nameToId('amazon_s3');

npm run test

TODO:

Get tests passing
Test on React Native

laurent22 · 2020-03-21T10:57:27Z

Thanks @alexchee, the code looks pretty good already. What test is failing at the moment?

laurent22 · 2020-03-21T10:59:54Z

ReactNativeClient/lib/shim-init-node.js

@@ -357,6 +357,16 @@ function shimInit() {
 		return Buffer.byteLength(string, 'utf-8');
 	};

+	shim.readFileToBlob = async function(path) {


That should be in fsdriver. But what is the purpose of it? It seems similar to fsDriver.readFile?

Glad you brought this up, I wasn't sure if I needed to handle deal with binary files on React Native differently. But readFile would be simplier.

alexchee · 2020-03-21T21:00:28Z

ReactNativeClient/lib/file-api-driver-amazon-s3.js

+				if (!options.path) throw new Error('get: target options.path is missing');
+
+				// TODO: check if this ever hits on RN
+				await shim.fsDriver().writeBinaryFile(options.path, output);


@laurent22 I have a question here for React Native, how should the driver handle get for files on RN? writeBinaryFile will raise an exception on the RN shim.

alexchee · 2020-03-21T21:10:09Z

Thanks @alexchee, the code looks pretty good already. What test is failing at the moment?

It's varies between 5 and 45 failed tests each run. I think I know my issue. S3 put are not transactional and it's not guaranteed to be there when the request is finished.
I can add some code there to wait for the file to exist before resolving a successful put response.

here are the current fails:

Failures:
1) synchronizer should not download resources over the limit
  Message:
    Expected 1 to be 2.
  Stack:
    Error: Expected 1 to be 2.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:1324:38
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

2) synchronizer should clear a lock if it was created by the same app as the current one
  Message:
    Expected 0 to be 1.
  Stack:
    Error: Expected 0 to be 1.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:1538:54
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

3) synchronizer should not wipe out user data when syncing with an empty target
  Message:
    Expected 2 to be 10.
  Stack:
    Error: Expected 2 to be 10.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:1492:37
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

4) synchronizer should sync resources
  Message:
    Expected false to be true.
  Stack:
    Error: Expected false to be true.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:878:60
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

5) synchronizer should not upload any item if encryption was enabled, and items have not been decrypted, and then encryption disabled
  Message:
    Expected false to be true.
  Stack:
    Error: Expected false to be true.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:1025:47
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

6) synchronizer should not consider it is a conflict if neither the title nor body of the note have changed
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

7) synchronizer should delete resources
  Message:
    Expected 0 to be 1.
  Stack:
    Error: Expected 0 to be 1.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:947:31
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

8) synchronizer should not upload a resource if it has not been fetched yet
  Message:
    Expected 0 to be 1.
  Stack:
    Error: Expected 0 to be 1.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:1352:44
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

9) synchronizer should delete remote notes
  Message:
    Expected 0 to be 1.
  Stack:
    Error: Expected 0 to be 1.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:258:26
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

10) synchronizer should encrypt remote resources after encryption has been enabled
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

11) synchronizer should not sync notes with conflicts
  Message:
    Expected 0 to be 1.
  Stack:
    Error: Expected 0 to be 1.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:592:26
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

12) synchronizer should encrypt existing notes too when enabling E2EE
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

13) synchronizer should upload decrypted items to sync target after encryption disabled
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

14) synchronizer should update remote items but not pull remote changes
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

15) synchronizer should handle case when new rev is created on client, then older rev arrives later via sync
  Message:
    Unhandled promise rejection: Error: No such note: 168a10b2130f4f34ac95a2c507fc7afa
  Stack:
    Error: No such note: 168a10b2130f4f34ac95a2c507fc7afa
        at Function.noteIsOlderThan (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/models/Note.js:555:17)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)
        at async RevisionService.isOldNote (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/services/RevisionService.js:41:17)

16) synchronizer should set the resource file size if it is missing
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)

17) synchronizer should delete local notes
  Message:
    Expected 3 to be 2.
  Stack:
    Error: Expected 3 to be 2.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:303:24
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

18) synchronizer should sync tags
  Message:
    Expected false to be true.
  Stack:
    Error: Expected false to be true.
        at <Jasmine>
        at shoudSyncTagTest (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:547:23)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

19) synchronizer should update remote items
  Message:
    Expected 2 to be 0.
  Stack:
    Error: Expected 2 to be 0.
        at <Jasmine>
        at localNotesFoldersSameAsRemote (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:66:25)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)
  Message:
    Expected Error to be null. Tip: To check for deep equality, use .toEqual() instead of .toBe().
  Stack:
    Error: Expected Error to be null. Tip: To check for deep equality, use .toEqual() instead of .toBe().
        at <Jasmine>
        at localNotesFoldersSameAsRemote (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:85:16)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

20) synchronizer should not save revisions when updating a note via sync
  Message:
    Unhandled promise rejection: Error: No such note: f4d4dda636fd4b83a171ab48dc6ce485
  Stack:
    Error: No such note: f4d4dda636fd4b83a171ab48dc6ce485
        at Function.noteIsOlderThan (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/models/Note.js:555:17)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)
        at async RevisionService.isOldNote (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/services/RevisionService.js:41:17)

21) synchronizer should decrypt the resource metadata, but not try to decrypt the file, if it is not present
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)

22) synchronizer should resolve note conflicts
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

23) synchronizer should not try to delete on remote conflicted notes that have been deleted
  Message:
    Unhandled promise rejection: Error: No such note: 96bb11d2cdca40e495c16bf31c62d64c
  Stack:
    Error: No such note: 96bb11d2cdca40e495c16bf31c62d64c
        at Function.noteIsOlderThan (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/models/Note.js:555:17)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)
        at async RevisionService.isOldNote (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/services/RevisionService.js:41:17)

24) synchronizer should create remote items with UTF-8 content
  Message:
    Expected 1 to be 0.
  Stack:
    Error: Expected 1 to be 0.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:607:31
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)
  Message:
    Expected 2 to be 0.
  Stack:
    Error: Expected 2 to be 0.
        at <Jasmine>
        at localNotesFoldersSameAsRemote (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:66:25)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)
  Message:
    Expected Error to be null. Tip: To check for deep equality, use .toEqual() instead of .toBe().
  Stack:
    Error: Expected Error to be null. Tip: To check for deep equality, use .toEqual() instead of .toBe().
        at <Jasmine>
        at localNotesFoldersSameAsRemote (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:85:16)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

25) synchronizer should update local items
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

26) synchronizer should enable encryption automatically when downloading new master key (and none was previously available)
  Message:
    Expected false to be true.
  Stack:
    Error: Expected false to be true.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:775:47
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

27) synchronizer should resolve folders conflicts
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

28) synchronizer should not save revisions when an item_change has been generated as a result of a sync
  Message:
    Unhandled promise rejection: Error: No such note: f79d09c48b1d4de4a6178db09cd3d10a
  Stack:
    Error: No such note: f79d09c48b1d4de4a6178db09cd3d10a
        at Function.noteIsOlderThan (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/models/Note.js:555:17)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)
        at async RevisionService.isOldNote (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/services/RevisionService.js:41:17)

29) synchronizer should encryt resources
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)

30) synchronizer should sync encrypted tags
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

31) synchronizer should allow duplicate folder titles
  Message:
    Expected 0 to be 1, 'Test has thrown an exception - see above error'.
  Stack:
    Error: Expected 0 to be 1, 'Test has thrown an exception - see above error'.
        at <Jasmine>
        at UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:393:15)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

32) synchronizer should handle resource download errors
  Message:
    Unhandled promise rejection: Error: Resource ID not provided
  Stack:
    Error: Resource ID not provided
        at Function.byResourceId (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/models/ResourceLocalState.js:14:26)
        at Function.localState (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/models/Resource.js:222:29)
        at ResourceFetcher.startDownload_ (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/lib/services/ResourceFetcher.js:123:37)
        at <Jasmine>
        at processTicksAndRejections (internal/process/task_queues.js:97:5)

264 specs, 32 failures
Finished in 193.476 seconds

alexchee · 2020-03-22T03:47:49Z

After playing with this more, S3 isn't good enough for syncing notes. The reads are inconsistent, I'll need to redo this with another AWS service like DynamoDB.

I'll close it for now until I get it working with another AWS service. Thanks for the help @laurent22 .

roncanepa · 2020-03-29T15:41:45Z

Hi @alexchee , sorry for the delay, I didn't realize that the forums weren't notifying me of your responses.

@laurent22 , what's your take on the possibilities for s3-compatible storage at this point, given all the work that @alexchee has done? Still think it's doable?

I've been meaning to see if I can serve up s3 (from minio) via webdav without it being too hacky. May try that now.

laurent22 · 2020-03-29T16:13:47Z

I'd be ok with adding an S3 sync target to the app, but I guess it's not up to me as I don't know enough about it.

I'm surprised that reads are inconsistent actually, given how widely used S3 is, do you know exactly what the issue was @alexchee?

alexchee · 2020-03-29T18:29:02Z

The current issue is using s3 as a lock for syncing. S3 has eventual consistency (read-after-write), but it's not reliable on it's own for locking syncs between multiple devices.

S3 is ok for one device to write time with mutliple devices to only read and download. But, if there's multiple devices/processes trying to write and obtain a lock, it'll behave like in the tests where it'll see files with unexpected timestamps because another process/test has written to that same file.

If something else can be using to determine locks for syncing with S3, this would make s3 a better storage option. Usually if people need to do something with syncing on s3, they assume there's only one source for that specific object being written or built something else around it to lock the s3 objects. I'm testing out something with dynamoDB (consistent reads/write but 40kb file limit) and S3 but that seems like a hassle for most users. Any ideas here?

laurent22 · 2020-03-29T18:49:15Z

Hmm yes Joplin sync has never really be tested with eventual consistency storage. I suppose it could work, it just means that changes will take a bit longer to be applied to all clients.

Test units however wouldn't work as it is because the test checks that the data has been written, and S3 will report that it has not.

Perhaps for testing purposes you could add a delay of x seconds after each put operation? That will ensure that following reads will be correct.

As for dynamoDB the 40kb limit would not be enough I think.

alexchee · 2020-03-30T20:06:36Z

I agree about dynamoDB.

So running each of the fail tests individually and deleting the bucket after each run makes most of the test pass. Only 3 tests are failing:

1) synchronizer should set the resource file size if it is missing
  Message:
    Expected 4725 to be 2720.
  Stack:
    Error: Expected 4725 to be 2720.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:930:19
        at processTicksAndRejections (internal/process/task_queues.js:97:5)
        at async UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:387:4)

$ npm run test -- --filter="synchronizer should sync resources"

> joplin@1.0.161 test /Users/alexchee/projects/joplin-fork/CliClient
> gulp buildTests -L && jasmine --config=tests/support/jasmine.json "--filter=synchronizer should sync resources"

Testing with sync target: amazon_s3
Randomized with seed 36668
Started
F

Failures:
1) synchronizer should sync resources
  Message:
    Expected false to be true.
  Stack:
    Error: Expected false to be true.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:878:60
        at processTicksAndRejections (internal/process/task_queues.js:97:5)
        at async UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:387:4)

$ npm run test -- --filter="synchronizer should clear a lock if it was created by the same app as the current one"

> joplin@1.0.161 test /Users/alexchee/projects/joplin-fork/CliClient
> gulp buildTests -L && jasmine --config=tests/support/jasmine.json "--filter=synchronizer should clear a lock if it was created by the same app as the current one"

Testing with sync target: amazon_s3
Randomized with seed 63429
Started
F

Failures:
1) synchronizer should clear a lock if it was created by the same app as the current one
  Message:
    Expected 0 to be 1.
  Stack:
    Error: Expected 0 to be 1.
        at <Jasmine>
        at /Users/alexchee/projects/joplin-fork/CliClient/tests-build/synchronizer.js:1538:54
        at processTicksAndRejections (internal/process/task_queues.js:97:5)
        at async UserContext.<anonymous> (/Users/alexchee/projects/joplin-fork/CliClient/tests-build/test-utils.js:387:4)

My guess is my delete method probably needs to be recursive with directories like .lock. I'll have to dig into why synchronizer should set the resource file size if it is missing is returning different file sizes.

alexchee · 2020-04-02T00:00:08Z

Ok, I got it working and the tests passing. I found out my implementation of FileApi list was not what was expected. I was returning the full path for path. I wrote some Tests to test FileApi in case anyone else in the future wants to add a new SyncTarget.

alexchee · 2020-04-02T00:03:49Z

I still an uncertain about if will work on ReactNative because I do some Buffer reading and writing when the target/source is file. I'll add comments in the diff in the 2 places I'm unsure about.

alexchee · 2020-04-02T00:04:47Z

ReactNativeClient/lib/file-api-driver-amazon-s3.js

+
+	async s3UploadFileFrom(path, key) {
+		if (!shim.fsDriver().exists(path)) throw new Error('s3UploadFileFrom: file does not exist');
+		const body = await shim.fsDriver().readFile(path, 'Buffer');


This is for put with options.source === 'file', not sure if this will break on ReactNative/mobile apps.

alexchee · 2020-04-13T15:25:07Z

I'm not sure why Travis is failing, the logs show the tests passing. I did do some file operations in the new tests, would that impact travis?

alexchee · 2020-05-01T17:03:23Z

looks like it's failing with NoteHistory test, after I rebased with master:

Failures:
1) integration_ForwardBackwardNoteHistory should save history when navigating through notes
  Message:
    Expected undefined to equal [  ].
  Stack:
    Error: Expected undefined to equal [  ].
        at <Jasmine>
        at asyncTest (/Users/travis/build/laurent22/joplin/CliClient/tests-build/feature_ForwardBackwardNoteHistory.js:41:38)
        at <Jasmine>
2) integration_ForwardBackwardNoteHistory should save history when navigating through notebooks
  Message:
    Expected undefined to equal [  ].
  Stack:
    Error: Expected undefined to equal [  ].
        at <Jasmine>
        at asyncTest (/Users/travis/build/laurent22/joplin/CliClient/tests-build/feature_ForwardBackwardNoteHistory.js:97:38)
        at <Jasmine>

Any advice on where I should look to fix this?

laurent22 · 2020-05-09T09:50:05Z

@alexchee, there was some problem on the test units on master, but it should be fixed now if you rebase.

Do you currently use the S3 sync target on desktop? If it works, I guess we could release it for desktop only for now and see if users might be interested. If they are, we could test on mobile and enable for it too. What do you think?

alexchee · 2020-05-09T15:38:44Z

That sounds like a good step. I currently use Dropbox sync so I can have it on my desktop and mobile, but I'll switch it over to S3 for my desktops to get more usage.
I'm eventually going to play with React Native on another project. Hopefully, I can help test it on mobile, when I get comfortable with RN.

matchett808-gh · 2020-07-14T14:32:48Z

It would be good if this was able to be configured to work with S3 compatible API's like Minio as well

alexchee · 2020-07-14T17:55:59Z

I like that idea to get more people to use this feature. But I have not used minIO. How easy would it be to adapt that API to this and would it require people to use minIO on their S3 buckets?

matchett808-gh · 2020-07-14T18:09:44Z

Minio allows a user to run their own S3 compatible server. So for example you can use a client like Boto3 to connect to either AWS S3 or to a minio installation.The only changes will likely be in configuration - allowing the user to specify the _entire_ URL etcI can try have a hack at it later this week and see if I can make it work.On 14 Jul 2020 18:56, alexchee <notifications@github.com> wrote: I like that idea to get more people to use this feature. But I have not used minIO. How easy would it be to adapt that API to this and would it require people to use minIO on their S3 buckets? —You are receiving this because you commented.Reply to this email directly, view it on GitHub, or unsubscribe.

matchett808-gh · 2020-07-14T18:13:44Z

Check this link, minio with the same package you have used: https://docs.min.io/docs/how-to-use-aws-sdk-for-javascript-with-minio-server.html Yeah, looking at your code - it looks like it just needs additional config fieldsOn 14 Jul 2020 19:09, iain@matchdev.co.uk wrote:Minio allows a user to run their own S3 compatible server. So for example you can use a client like Boto3 to connect to either AWS S3 or to a minio installation. The only changes will likely be in configuration - allowing the user to specify the _entire_ URL etcI can try have a hack at it later this week and see if I can make it work

roncanepa · 2020-07-14T18:33:05Z

As long as it's "s3-compatible" we should be most of the way there and then we can test it against the specific storage backends. For instance, I run Ceph at work and would hapilly run Joplin on my own personal Ceph cluster. S3 compatibility would open so many more (and better) storage options.

How can I help test the builds? Are there releases for your branch?

alexchee · 2020-07-14T20:08:16Z

I think @laurent22 was waiting for people that wanted to use it before releasing it to test out.

matchett808-gh · 2020-07-14T20:12:46Z

Is this branch in a working state ATM?

alexchee · 2020-07-14T20:15:31Z

It is working. And the conflicts with master seems to be resolvable.

matchett808-gh · 2020-07-14T20:19:16Z

Cool, I'm rebuilding my work laptop 2m, I'll see about spinning it up and testing it with minio

* added tests for FileApiDriver

alexchee · 2020-07-14T20:25:59Z

I just resolved the conflict and force pushed out the resolved code. You might want to pull down the new branch

matchett808-gh · 2020-07-14T21:27:33Z

(please be gentle - been a long while since I've done anything React or Electron based)

alexchee#1

it didn't take much - but the PR appears to sync with minio :)

matchett808-gh · 2020-07-14T21:31:41Z

Headlines are:

Need to specify the Url of the server for minio, minio also requires a couple of extra parameters
I can confirm syncing works - 'url' is the base url of the server
as with S3, you need to set the permissions (this will vary between Ceph, minio etc)

laurent22 · 2020-07-15T09:24:22Z

@alexchee, thanks for the pull request. I've checked again and don't see any issue so let's merge. For now I've added (Beta) to the sync target label to make it clear we might need a bit more real-life testing, but I expect we can make it official quite soon.

tessus added the WIP label Mar 20, 2020

laurent22 reviewed Mar 21, 2020

View reviewed changes

alexchee commented Mar 21, 2020

View reviewed changes

alexchee closed this Mar 22, 2020

alexchee reopened this Apr 2, 2020

alexchee commented Apr 2, 2020

View reviewed changes

laurent22 added the high High priority issues label Apr 30, 2020

alexchee force-pushed the aws_s3_sync branch 2 times, most recently from 7419215 to 1f0d299 Compare May 1, 2020 16:42

alexchee force-pushed the aws_s3_sync branch from 1f0d299 to 215d1ce Compare May 9, 2020 15:37

alexchee force-pushed the aws_s3_sync branch from 215d1ce to e5e1bd5 Compare May 20, 2020 14:30

adds AWS S3 sync support

7ee33f7

* added tests for FileApiDriver

alexchee force-pushed the aws_s3_sync branch from e5e1bd5 to 7ee33f7 Compare July 14, 2020 20:25

Update SyncTargetAmazonS3.js

5688d3a

laurent22 merged commit 9a55afe into laurent22:master Jul 15, 2020

alexchee deleted the aws_s3_sync branch July 15, 2020 22:49

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add AWS S3 sync #2815

Add AWS S3 sync #2815

alexchee commented Mar 20, 2020

laurent22 commented Mar 21, 2020

laurent22 Mar 21, 2020

alexchee Mar 21, 2020

alexchee Mar 21, 2020

alexchee commented Mar 21, 2020

alexchee commented Mar 22, 2020

roncanepa commented Mar 29, 2020

laurent22 commented Mar 29, 2020

alexchee commented Mar 29, 2020

laurent22 commented Mar 29, 2020

alexchee commented Mar 30, 2020

alexchee commented Apr 2, 2020

alexchee commented Apr 2, 2020

alexchee Apr 2, 2020

alexchee commented Apr 13, 2020

alexchee commented May 1, 2020

laurent22 commented May 9, 2020

alexchee commented May 9, 2020 •

edited

Loading

matchett808-gh commented Jul 14, 2020

alexchee commented Jul 14, 2020

matchett808-gh commented Jul 14, 2020 via email

matchett808-gh commented Jul 14, 2020 via email •

edited

Loading

roncanepa commented Jul 14, 2020

alexchee commented Jul 14, 2020

matchett808-gh commented Jul 14, 2020 via email •

edited

Loading

alexchee commented Jul 14, 2020

matchett808-gh commented Jul 14, 2020 via email •

edited

Loading

alexchee commented Jul 14, 2020

matchett808-gh commented Jul 14, 2020

matchett808-gh commented Jul 14, 2020

laurent22 commented Jul 15, 2020

Add AWS S3 sync #2815

Add AWS S3 sync #2815

Conversation

alexchee commented Mar 20, 2020

Setup

Running

Testing

TODO:

laurent22 commented Mar 21, 2020

laurent22 Mar 21, 2020

Choose a reason for hiding this comment

alexchee Mar 21, 2020

Choose a reason for hiding this comment

alexchee Mar 21, 2020

Choose a reason for hiding this comment

alexchee commented Mar 21, 2020

alexchee commented Mar 22, 2020

roncanepa commented Mar 29, 2020

laurent22 commented Mar 29, 2020

alexchee commented Mar 29, 2020

laurent22 commented Mar 29, 2020

alexchee commented Mar 30, 2020

alexchee commented Apr 2, 2020

alexchee commented Apr 2, 2020

alexchee Apr 2, 2020

Choose a reason for hiding this comment

alexchee commented Apr 13, 2020

alexchee commented May 1, 2020

laurent22 commented May 9, 2020

alexchee commented May 9, 2020 • edited Loading

matchett808-gh commented Jul 14, 2020

alexchee commented Jul 14, 2020

matchett808-gh commented Jul 14, 2020 via email

matchett808-gh commented Jul 14, 2020 via email • edited Loading

roncanepa commented Jul 14, 2020

alexchee commented Jul 14, 2020

matchett808-gh commented Jul 14, 2020 via email • edited Loading

alexchee commented Jul 14, 2020

matchett808-gh commented Jul 14, 2020 via email • edited Loading

alexchee commented Jul 14, 2020

matchett808-gh commented Jul 14, 2020

matchett808-gh commented Jul 14, 2020

laurent22 commented Jul 15, 2020

alexchee commented May 9, 2020 •

edited

Loading

matchett808-gh commented Jul 14, 2020 via email •

edited

Loading

matchett808-gh commented Jul 14, 2020 via email •

edited

Loading

matchett808-gh commented Jul 14, 2020 via email •

edited

Loading