Tools: Use Yarn 3 to manage monorepo (#5833)

.eslintignore

@@ -5,13 +5,16 @@ _releases/
 *.min.js
 **/commands/index.ts
 **/node_modules/
+.yarn/
 Assets/
 docs/
 highlight.pack.js
 Modules/TinyMCE/IconPack/postinstall.js
 Modules/TinyMCE/JoplinLists/
 Modules/TinyMCE/langs/
 node_modules/
+packages/app-desktop/build/
+packages/app-cli/tests/test data/
 packages/app-cli/build
 packages/app-cli/build/
 packages/app-cli/locales
@@ -1509,9 +1512,6 @@ packages/lib/services/plugins/api/types.js.map
 packages/lib/services/plugins/reducer.d.ts
 packages/lib/services/plugins/reducer.js
 packages/lib/services/plugins/reducer.js.map
-packages/lib/services/plugins/sandboxProxy.d.ts
-packages/lib/services/plugins/sandboxProxy.js
-packages/lib/services/plugins/sandboxProxy.js.map
 packages/lib/services/plugins/utils/createViewHandle.d.ts
 packages/lib/services/plugins/utils/createViewHandle.js
 packages/lib/services/plugins/utils/createViewHandle.js.map
@@ -1941,9 +1941,6 @@ packages/tools/generate-images.js.map
 packages/tools/git-changelog.d.ts
 packages/tools/git-changelog.js
 packages/tools/git-changelog.js.map
-packages/tools/lerna-add.d.ts
-packages/tools/lerna-add.js
-packages/tools/lerna-add.js.map
 packages/tools/release-android.d.ts
 packages/tools/release-android.js
 packages/tools/release-android.js.map

.github/scripts/run_ci.sh

@@ -46,13 +46,14 @@ echo "IS_MACOS=$IS_MACOS"
 
 echo "Node $( node -v )"
 echo "Npm $( npm -v )"
+echo "Yarn $( yarn -v )"
 
 # =============================================================================
 # Install packages
 # =============================================================================
 
 cd "$ROOT_DIR"
-npm install
+yarn install
 
 # =============================================================================
 # Run test units. Only do it for pull requests and dev branch because we don't
@@ -78,7 +79,7 @@ if [ "$IS_PULL_REQUEST" == "1" ] || [ "$IS_DEV_BRANCH" = "1" ]; then
 	#
 	# https://stackoverflow.com/questions/38558989
 	export NODE_OPTIONS="--max-old-space-size=4096"
-	npm run test-ci
+	yarn run test-ci
 	testResult=$?
 	if [ $testResult -ne 0 ]; then
 		exit $testResult
@@ -93,7 +94,7 @@ fi
 if [ "$IS_PULL_REQUEST" == "1" ] || [ "$IS_DEV_BRANCH" = "1" ]; then
 	echo "Step: Running linter..."
 
-	npm run linter-ci ./
+	yarn run linter-ci ./
 	testResult=$?
 	if [ $testResult -ne 0 ]; then
 		exit $testResult
@@ -166,12 +167,12 @@ cd "$ROOT_DIR/packages/app-desktop"
 
 if [[ $GIT_TAG_NAME = v* ]]; then
 	echo "Step: Building and publishing desktop application..."
-	USE_HARD_LINKS=false npm run dist
+	USE_HARD_LINKS=false yarn run dist
 elif [[ $GIT_TAG_NAME = server-v* ]] && [[ $IS_LINUX = 1 ]]; then
 	echo "Step: Building Docker Image..."
 	cd "$ROOT_DIR"
-	npm run buildServerDocker -- --tag-name $GIT_TAG_NAME --push-images
+	yarn run buildServerDocker -- --tag-name $GIT_TAG_NAME --push-images
 else
 	echo "Step: Building but *not* publishing desktop application..."
-	USE_HARD_LINKS=false npm run dist -- --publish=never
+	USE_HARD_LINKS=false yarn run dist -- --publish=never
 fi

.github/workflows/github-actions-main.yml

@@ -49,6 +49,11 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: '16'
+
+      - name: Install Yarn
+        run: |
+          # https://yarnpkg.com/getting-started/install
+          corepack enable
       
       # Login to Docker only if we're on a server release tag. If we run this on
       # a pull request it will fail because the PR doesn't have access to
@@ -80,6 +85,44 @@ jobs:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
           IS_CONTINUOUS_INTEGRATION: 1
         run: |
-          npm install
+          yarn install
           cd packages/app-desktop
-          npm run dist
+          yarn run dist
+
+  ServerDockerImage:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    steps:
+
+      - name: Install Docker Engine
+        run: |
+          sudo apt-get install -y apt-transport-https
+          sudo apt-get install -y ca-certificates
+          sudo apt-get install -y curl
+          sudo apt-get install -y gnupg
+          sudo apt-get install -y lsb-release
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+          echo \
+              "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
+              $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update || true
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+
+      - name: Install Yarn
+        run: |
+          # https://yarnpkg.com/getting-started/install
+          corepack enable
+
+      - name: Build Docker Image
+        run: |
+          yarn install
+          yarn run buildServerDocker --tag-name server-v0.0.0
+          

.gitignore

@@ -51,6 +51,17 @@ lerna-debug.log
 .env
 docs/**/*.mustache
 
+# Yarn stuff
+# https://yarnpkg.com/getting-started/qa#which-files-should-be-gitignored
+
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
 # AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
 packages/app-cli/app/LinkSelector.d.ts
 packages/app-cli/app/LinkSelector.js
@@ -1492,9 +1503,6 @@ packages/lib/services/plugins/api/types.js.map
 packages/lib/services/plugins/reducer.d.ts
 packages/lib/services/plugins/reducer.js
 packages/lib/services/plugins/reducer.js.map
-packages/lib/services/plugins/sandboxProxy.d.ts
-packages/lib/services/plugins/sandboxProxy.js
-packages/lib/services/plugins/sandboxProxy.js.map
 packages/lib/services/plugins/utils/createViewHandle.d.ts
 packages/lib/services/plugins/utils/createViewHandle.js
 packages/lib/services/plugins/utils/createViewHandle.js.map
@@ -1924,9 +1932,6 @@ packages/tools/generate-images.js.map
 packages/tools/git-changelog.d.ts
 packages/tools/git-changelog.js
 packages/tools/git-changelog.js.map
-packages/tools/lerna-add.d.ts
-packages/tools/lerna-add.js
-packages/tools/lerna-add.js.map
 packages/tools/release-android.d.ts
 packages/tools/release-android.js
 packages/tools/release-android.js.map

.yarnrc.yml

@@ -0,0 +1,9 @@
+nmHoistingLimits: workspaces
+
+nodeLinker: node-modules
+
+plugins:
+  - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
+    spec: "@yarnpkg/plugin-workspace-tools"
+
+yarnPath: .yarn/releases/yarn-3.1.1.cjs

BUILD.md

@@ -1,6 +1,6 @@
 # Building the applications
 
-The Joplin source code is hosted on a [monorepo](https://en.wikipedia.org/wiki/Monorepo) managed by Lerna. The usage of Lerna is mostly transparent as the needed commands have been moved to the root package.json and thus are invoked for example when running `npm install` or `npm run watch`. The main thing to know about Lerna is that it links the packages in the monorepo using `npm link`, so if you check the node_modules directory you will see links instead of actual directories for certain packages. This is something to keep in mind as these links can cause issues in some cases.
+The Joplin source code is hosted on a [monorepo](https://en.wikipedia.org/wiki/Monorepo) and is managed using Yarn workspaces (as well as Lerna for publishing the packages).
 
 The list of the main sub-packages is below:
 
@@ -18,30 +18,30 @@ There are also a few forks of existing packages under the "fork-*" name.
 
 ## Required dependencies
 
-- Install node 14+ - https://nodejs.org/en/
+- Install node 16+ - https://nodejs.org/en/
 - macOS: Install Cocoapods - `brew install cocoapods`
-- Windows: Install Windows Build Tools - `npm install -g windows-build-tools --vs2015`
+- Windows: Install Windows Build Tools - `yarn install -g windows-build-tools --vs2015`
 - Linux: Install dependencies - `sudo apt install build-essential libnss3 libsecret-1-dev python rsync`
 
 ## Building
 
 Before doing anything else, from the root of the project, run:
 
-	npm install
+	yarn install
 
 Then you can test the various applications:
 
 ## Testing the desktop application
 
 	cd packages/app-desktop
-	npm start
+	yarn start
 
 You can also run it under WSL 2. To do so, [follow these instructions](https://www.beekeeperstudio.io/blog/building-electron-windows-ubuntu-wsl2) to setup your environment.
 
 ## Testing the Terminal application
 
 	cd packages/app-cli
-	npm start
+	yarn start
 
 ## Testing the Mobile application
 
@@ -54,36 +54,28 @@ Then, for **Android**:
 
 On **iOS**, open the file `ios/Joplin.xcworkspace` on XCode and run the app from there.
 
-Normally the **bundler** should start automatically with the application. If it doesn't, run `npm start` from `packages/app-mobile`.
+Normally the **bundler** should start automatically with the application. If it doesn't, run `yarn start` from `packages/app-mobile`.
 
 ## Building the clipper
 
 	cd packages/app-clipper/popup
-	npm run watch # To watch for changes
+	yarn run watch # To watch for changes
 
 To test the extension please refer to the relevant pages for each browser: [Firefox](https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Your_first_WebExtension#Trying_it_out) / [Chrome](https://developer.chrome.com/docs/extensions/mv3/getstarted/). Please note that the extension in dev mode will only connect to a dev instance of the desktop app (and vice-versa).
 
 ## Watching files
 
 To make changes to the application, you'll need to rebuild any TypeScript file you've changed. The simplest way to do this is to watch for changes from the root of the project. Simply run this command, and it should take care of the rest:
 
-	npm run watch
+	yarn run watch
 
-Running `npm run tsc` would have the same effect, but without watching.
+Running `yarn run tsc` would have the same effect, but without watching.
 
 ## Running an application with additional parameters
 
-You can specify additional parameters when running the desktop or CLI application. To do so, add `--` to the `npm start` command, followed by your flags. For example:
+You can specify additional parameters when running the desktop or CLI application. To do so, add `--` to the `yarn start` command, followed by your flags. For example:
 
-	npm start -- --debug
-
-## Adding a new dependency
-
-Since Joplin uses Lerna, adding a new dependency should not be done using `npm i -s ...`. Instead you should use the `lerna add` command, which will take care of adding the package while handling the linked packages correctly. For example, to add the package "leftpad" to the "app-desktop" sub-package, you would run:
-
-	npx lerna add leftpad --scope=@joplin/app-desktop
-
-Note that you should most likely always specify a scope because otherwise it will add the package to all the sub-packages.
+	yarn start -- --debug
 
 ## TypeScript
 
@@ -95,14 +87,14 @@ If you'd like to auto-reload the desktop app on changes rather than having to qu
 
 ```sh
 cd packages/app-desktop
-watchman-make -p '**/*.js' '**/*.jsx' --run "npm start"
+watchman-make -p '**/*.js' '**/*.jsx' --run "yarn start"
 ```
 
-It still requires you to quit the application each time you want it to rebuild, but at least you don't have to re-run `"npm start"` each time. Here's what the workflow loop looks like in practice:
+It still requires you to quit the application each time you want it to rebuild, but at least you don't have to re-run `"yarn start"` each time. Here's what the workflow loop looks like in practice:
 
 1. Edit and save files in your text editor.
 2. Switch to the Electron app and <kbd>cmd</kbd>+<kbd>Q</kbd> to quit it.
-3. `watchman` immediately restarts the app for you (whereas usually you'd have to switch back to the terminal, type `"npm start"`, and hit enter).
+3. `watchman` immediately restarts the app for you (whereas usually you'd have to switch back to the terminal, type `"yarn start"`, and hit enter).
 
 # Troubleshooting
 

CONTRIBUTING.md

@@ -44,7 +44,7 @@ Building the apps is relatively easy - please [see the build instructions](https
 
 ## Coding style
 
-Coding style is enforced by a pre-commit hook that runs eslint. This hook is installed whenever running `npm install` on any of the application directory. If for some reason the pre-commit hook didn't get installed, you can manually install it by running `npm install` at the root of the repository.
+Coding style is enforced by a pre-commit hook that runs eslint. This hook is installed whenever running `yarn install` on any of the application directory. If for some reason the pre-commit hook didn't get installed, you can manually install it by running `yarn install` at the root of the repository.
 
 For new React components, please use [React Hooks](https://reactjs.org/docs/hooks-intro.html). For new code in general, please use TypeScript. Even if you are modifying a file that was originally in JavaScript you should ideally convert it first to TypeScript before modifying it. Doing so is relatively easy and it helps maintain code quality.
 
@@ -61,26 +61,26 @@ When submitting a pull request for a new feature or a bug fix, please add automa
 The tests are under packages/app-cli/tests. To get them running, you first need to build the CLI app:
 
 ```sh
-npm install
+yarn install
 cd packages/app-cli
 ```
 
 To run all the test units:
 
 ```sh
-npm test
+yarn test
 ```
 
 To run just one particular file:
 
 ```sh
-npm test -- --filter=markdownUtils # Don't add the .js extension
+yarn test -- --filter=markdownUtils # Don't add the .js extension
 ```
 
 To filter tests. For example, to run all the test units that contain "should handle conflict" in their description:
 
 ```sh
-npm test -- --filter="should handle conflict"
+yarn test -- --filter="should handle conflict"
 ```
 
 ## About abandoned pull requests