From 1ff6a8d532b806695984ba01e588405d03e5b133 Mon Sep 17 00:00:00 2001 From: Andrey Nikiforov Date: Sun, 16 Jun 2024 11:15:29 -0700 Subject: [PATCH] add documentation (#869) --- .github/workflows/build-docs.yml | 32 +++++++++ .github/workflows/publish-docs.yml | 21 ++++++ .github/workflows/quality-checks.yml | 2 +- .github/workflows/refresh-docs.yml | 18 ++++++ .gitignore | 2 + CONTRIBUTING.md | 8 +++ README.md | 4 ++ docs/authentication.md | 49 ++++++++++++++ docs/conf.py | 30 +++++++++ docs/index.md | 18 ++++++ docs/install.md | 97 ++++++++++++++++++++++++++++ docs/mode.md | 7 ++ docs/naming.md | 38 +++++++++++ docs/raw.md | 41 ++++++++++++ docs/size.md | 31 +++++++++ pyproject.toml | 5 +- 16 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-docs.yml create mode 100644 .github/workflows/publish-docs.yml create mode 100644 .github/workflows/refresh-docs.yml create mode 100644 docs/authentication.md create mode 100644 docs/conf.py create mode 100644 docs/index.md create mode 100644 docs/install.md create mode 100644 docs/mode.md create mode 100644 docs/naming.md create mode 100644 docs/raw.md create mode 100644 docs/size.md diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml new file mode 100644 index 000000000..37e5f56a9 --- /dev/null +++ b/.github/workflows/build-docs.yml @@ -0,0 +1,32 @@ +name: "Render docs" + +on: workflow_call + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Pages + id: pages + uses: actions/configure-pages@v5 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Dev dependencies + run: | + pip3 install --disable-pip-version-check -e .[dev] + + - name: Build HTML + run: | + sphinx-build docs docs/_build/html + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/_build/html/ + diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml new file mode 100644 index 000000000..fa151d950 --- /dev/null +++ b/.github/workflows/publish-docs.yml @@ -0,0 +1,21 @@ +name: "Publish docs to GH pages" + +on: workflow_call + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +jobs: + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + name: Deploy + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 \ No newline at end of file diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 5a8c340f6..ac77adc93 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -24,7 +24,7 @@ jobs: with: concurrent_skipping: 'same_content_newer' skip_after_successful_duplicate: 'true' - paths_ignore: '["**/*.md", "examples/**"]' + paths_ignore: '["**/*.md", "examples/**", "docs/**"]' do_not_skip: '["workflow_dispatch", "schedule"]' lint: diff --git a/.github/workflows/refresh-docs.yml b/.github/workflows/refresh-docs.yml new file mode 100644 index 000000000..47963e6d5 --- /dev/null +++ b/.github/workflows/refresh-docs.yml @@ -0,0 +1,18 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Refresh Docs + +on: + push: + tags: + - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 + workflow_dispatch: + +jobs: + build_docs: + uses: ./.github/workflows/build-docs.yml + + publish_docs: + needs: [build_docs] + uses: ./.github/workflows/publish-docs.yml diff --git a/.gitignore b/.gitignore index 672e3c40c..4d3fe376a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ htmlcov .venv venv *.spec +# docs +docs/_build # exclude since there is no js development, just for local testing npm packaging node_modules/ /package.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9924cf8ab..3fd57223b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -146,6 +146,14 @@ docker build -t icloudpd_dev_ . ``` Note: If you work with devcontainers, you most likely need to run that command on the host system inside your source folder. +### Developing Documentation + +To compile docs and start web server with hot reloading: + +``` sh +sphinx-autobuild docs docs/_build/html +``` + ## How to write a unit test The unit tests are a very important asset of this project. Due to our 100% test coverage we can safely use great tools like [Dependabot](dependabot.com) and be sure that the implementation of a new feature or fixing of a bug does not lead to further issues. diff --git a/README.md b/README.md index e96a84cc0..37100eefb 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ There are three ways to run `icloudpd`: ## Features + + - Three modes of operation: - **Copy** - download new photos from iCloud (default mode) - **Sync** - download new photos from iCloud and delete local files that were removed in iCloud (`--auto-delete` option) @@ -27,6 +29,8 @@ There are three ways to run `icloudpd`: - Photo meta data (EXIF) updates (`--set-exif-datetime` option) - ... and many more (use `--help` option to get full list) + + ## Experimental Mode Some changes are added to the experimental mode before they graduate into the main package. [Details](EXPERIMENTAL.md) diff --git a/docs/authentication.md b/docs/authentication.md new file mode 100644 index 000000000..579294bea --- /dev/null +++ b/docs/authentication.md @@ -0,0 +1,49 @@ +# iCloud Authentication + +## Multi Factor Authentication + +If your Apple account has two-factor authentication (multi-factor authentication, MFA) enabled, +you will be prompted for a code when you run the script. Two-factor authentication will expire after an interval set by Apple, +at which point you will have to re-authenticate. This interval is currently two months. Apple requires MFA for all new accounts. + +You can receive an email notification when two-factor authentication expires by passing the +`--smtp-username` and `--smtp-password` options. Emails will be sent to `--smtp-username` by default, +or you can send to a different email address with `--notification-email`. + +If you want to send notification emails using your Gmail account, and you have enabled two-factor authentication, you will need to generate an App Password at + +## FIDO + +Authentication to iCloud with hardware keys (FIDO) is not supported. + +## ADP + +Advanced Data Protection (ADP) for iCloud accounts is not supported because iCloudPD simulates web access, which is disabled with ADP. + +## Occasional Errors + +Some authentication errors may be resolved by clearing `.pycloud` subfolder in the user's home dir. [Example](https://github.com/icloud-photos-downloader/icloud_photos_downloader/issues/772#issuecomment-1950963522) + +## System Keyring + +You can store your password in the system keyring using the `icloud` command-line tool: + +``` +$ icloud --username jappleseed@apple.com +ICloud Password for jappleseed@apple.com: +Save password in keyring? (y/N) +``` + +If you have stored a password in the keyring, you will not be required to provide a password +when running the script. + +If you would like to delete a password stored in your system keyring, +you can clear a stored password using the `--delete-from-keyring` command-line option: + +``` sh +icloud --username jappleseed@apple.com --delete-from-keyring +``` + +```{note} +Use `icloud`, not `icloudpd` +``` \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 000000000..dae6d4ebf --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,30 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'icloudpd' +copyright = '2024, Contributors' +author = 'Contributors' +release = '1.19.1' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["myst_parser"] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +html_title = "icloudpd" +language = "en" +# language = 'Python' + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'furo' +# html_static_path = ['_static'] diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..ffa8f522e --- /dev/null +++ b/docs/index.md @@ -0,0 +1,18 @@ +# iCloud Photos Downloader + +A command-line tool to download your iCloud photos to local storage + +```{include} ../README.md +:start-after: +:end-before: +``` + +```{toctree} +:hidden: +install +authentication +naming +mode +size +raw +``` diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 000000000..6fb281718 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,97 @@ +# Install and Run + +There are three ways to run `icloudpd`: +1. Download executable for your platform from the Github [Releases](https://github.com/icloud-photos-downloader/icloud_photos_downloader/releases) and run it +1. Use package manager to install, update, and, in some cases, run ([Docker](#docker), [PyPI](#pypi), [AUR](#aur), [Npm](#npm)) +1. Build and run from the source + +(docker)= +## Docker + +```sh +docker run -it --rm --name icloudpd -v $(pwd)/Photos:/data -e TZ=America/Los_Angeles icloudpd/icloudpd:latest icloudpd --directory /data --username my@email.address --watch-with-interval 3600 +``` + +Image asset date will be convered to specified TZ and then used for creating folders (see `--folder-stucture` param) + +Synchronization logic can be adjusted with command-line parameters. Run the following to get full list: +``` sh +docker run -it --rm icloudpd/icloudpd:latest icloudpd --help +``` + +```{note} +On Windows: + +- use `%cd%` instead of `$(pwd)` +- or full path, e.g. `-v c:/photos/icloud:/data` +- only Linux containers are supported +``` + +```{note} + +Getting Docker: + +- On Windows and Mac Docker is available as [Docker Desktop](https://www.docker.com/products/docker-desktop/) app. + +- On Linux, Docker engine and client can be installed using platform package managers, e.g. [Installing on Ubuntu](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-20-04) + +- Appliance (e.g. NAS) will have their own way to install Docker engines and running containers - see manufacturer's instructions. +``` + +(pypi)= +## PyPI + +Install: +``` sh +pip install icloudpd +``` + +Run: + +``` sh +icloudpd --directory /data --username my@email.address --watch-with-interval 3600 +``` + +````{note} + +on Windows: + +``` sh +pip install icloudpd --user +``` + +Plus add `C:\Users\\AppData\Roaming\Python\Python\Scripts` to PATH. The exact path will be given at the end of `icloudpd` installation. +```` + +```{note} + +on MacOS: + +Add `/Users//Library/Python//bin` to PATH. The exact path will be given at the end of `icloudpd` installation. +``` + +(aur)= +## AUR + +AUR packages can be installed on Arch Linux. Installation can be done [manually](https://wiki.archlinux.org/title/Arch_User_Repository#Installing_and_upgrading_packages) or with the use of an [AUR helper](https://wiki.archlinux.org/title/AUR_helpers). + +The manual process would look like this: + +``` sh +git clone https://aur.archlinux.org/icloudpd-bin.git +cd icloudpd-bin +makepkg -sirc +``` + +With the use of the AUR helper e.g. [yay](https://github.com/Jguer/yay) the installation process would look like this: + +``` sh +yay -S icloudpd-bin +``` + +(npm)= +## NPM + +``` sh +npx --yes icloudpd --directory /data --username my@email.address --watch-with-interval 3600 +``` diff --git a/docs/mode.md b/docs/mode.md new file mode 100644 index 000000000..6c1889284 --- /dev/null +++ b/docs/mode.md @@ -0,0 +1,7 @@ +# Operation Modes + +`icloudpd` works in one of three modes of operation: + +- **Copy** - download new photos from iCloud (default mode) +- **Sync** - download new photos from iCloud and delete local files that were removed in iCloud (`--auto-delete` parameter) +- **Move** - download new photos from iCloud and delete photos in iCloud (`--delete-after-download` parameter) \ No newline at end of file diff --git a/docs/naming.md b/docs/naming.md new file mode 100644 index 000000000..3213b187a --- /dev/null +++ b/docs/naming.md @@ -0,0 +1,38 @@ +# File Naming + +Assets on iCloud have names. When downloading assets, `icloudpd` can adjust names. + +## Folder Structure + +`icloudpd` uses asset metadata (_created date_) to build folder hierarchy and it can be adjusted with `--folder-stucture` parameter. + +Specifying `--folder-structure none` will put all files into one folder. + +## Duplicates + +```{versionchanged} 1.19.0 +`--file-match-policy` parameter added and `name-id7` policy implemented +``` + +In large iCloud collections it is possible to have name collisions. To avoid collissions if files need to be downloaded into the same folder, use `--file-match-policy` parameter: +- add unique invariant asset identification suffix to the name (e.g. **"IMG_1234_QAZXSW.JPG"**) with `--file-match-policy name-id7` +- deduplicate by adding file size as a suffix (e.g. **"IMG_1234-67890.JPG"** for second asset); `--file-match-policy name-size-dedup-with-suffix` - it is default + +## Live Photos + +```{versionchanged} 1.18.0 +`--live-photo-mov-filename-policy` parameter added and `original` policy implemented +``` + +Live Photo assets have two components: still image and short video. `icloudpd` can download both and allows customizing file name of the video portion with `--live-photo-mov-filename-policy` parameter: + +- use video file name the same as still image with `original` policy; use `--file-match-policy name-id7` to avoid clashes of video file with other videos. +- use suffix from the still image with `suffix` policy: **"IMG_1234_HEVC.MOV"** for **"IMG_1234.HEIC"** still. This is default and works for HIEC still images only + +## Unicode + +```{versionchanged} 1.18.0 +`--keep-unicode-in-filenames` parameter flag added with default `false` +``` + +Unicode characters are stripted from file names for better compatibility. `icloudpd` can leave them when `--keep-unicode-in-filenames` is specified. diff --git a/docs/raw.md b/docs/raw.md new file mode 100644 index 000000000..d5efe14f9 --- /dev/null +++ b/docs/raw.md @@ -0,0 +1,41 @@ +# RAW Assets + +## Apple ProRAW/ProRes + +Apple supports shooting stills and videos in [DNG](https://en.wikipedia.org/wiki/Digital_Negative) format and +they can be downloaded by `icloudpd` as any other supported format. + +## Imported RAW images + +```{versionadded} 1.19.0 +``` + +RAW images from third party cameras can be imported into Apple Photos or uploaded to iCloud.com. +These type of assets can also be downloaded by `icloudpd`. The following formats are recognized: + +- Adobe DNG - same as Apple ProRAW +- Canon CR2, CR3, and CRW +- Sony ARW +- Fuji RAF +- Panasonic RW2 +- Nikon NRF, NEF +- Pentax PEF +- Olympus ORF + +## RAW+JPEG and JPEG+RAW + +```{versionadded} 1.19.0 +``` + +iCloud supports images with two represenations. `icloudpd` can download one or both representations. + +One represenation will be `original` [size](size) and another `alternative`. + +As of June 2024, icloud.com always shows assets with two representations as RAW+JPEG. Photo app on Mac +allows choosing which representation to treat as original, but it is not clear what that setting changes. + +`icloudpd` diambiguates behavior with `--align-raw` parameter: + +- *original* always treat RAW as original [size](size) +- *alternative* always treat RAW as alternative [size](size) +- *as-is* treat RAW as it is in icloud data diff --git a/docs/size.md b/docs/size.md new file mode 100644 index 000000000..4a5c2315b --- /dev/null +++ b/docs/size.md @@ -0,0 +1,31 @@ +# Asset Sizes + +## Basic Sizes + +Each asset in iCloud may have multiple sizes available for downloading: +- original +- medium +- thumb + +Size can be selected with `--size` parameter and multiple specifications are accepted (e.g. `--size original --size medium`). Default is `original`. + +If size parameter is specified, but it is not available for the asset in iCloud, then `original` is downloaded, unless `--force-size` is specified. + +Assets for sizes other than original will have suffix added to their name, e.g. `IMG-1234-medium.JPG`. + +## Special Sizes + +```{versionadded} 1.19.0 +``` + +Image edits are represented as a special size `adjusted`. Two common use cases related to edits: +- Download only edited version: use `--size adjusted` parameter +- Download edit and original: use `--size adjusted --size original` parameters. Edits will have `-adjusted` suffix added if file extension is the same as original. + +```{note} +Portraits are represented in iCloud as edits. +``` + +```{seealso} +[RAW](raw) assets use `alternative` size +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cd4d95eaf..40508a2b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,9 +43,12 @@ dependencies = [ [project.optional-dependencies] dev = [ "twine==4.0.2", - # 6+ breaks keyring https://github.com/AndreyNikiforov/keyring_test "pyinstaller==6.7.0", "wheel==0.42.0", + "furo==2024.5.6", + "Sphinx==7.3.7", + "sphinx-autobuild==2024.4.16", + "myst-parser==3.0.1" ] devlinux = [ "auditwheel==5.4.0",