CLI command to upload plugins/themes to AspireCloud (#42)

* feat: add meta/bin/push-to-aspirecloud * feat: upload to AC in chunks of 100 (configurable) * fix: add ASPIRECLOUD_ADMIN_API_* envars to docker-compose.yml * feat: add --after parameter to meta dump commands * feat: enable compression in push-to-aspirecloud * fix: respect the --update-all flag once again * docs: update README with AC integration instructions
aspirepress · Dec 28, 2024 · d9406d3 · d9406d3
1 parent afcc757
commit d9406d3
Show file tree

Hide file tree

Showing 10 changed files with 185 additions and 59 deletions.
diff --git a/.env b/.env
@@ -16,12 +16,6 @@
 
 ########## AspireSync Config
 
-# TODO: ditch log config in here and use symfony config
-# Log level may be one of: emergency | alert | critical | error | warning | notice | info | debug
-# LOG_FILE=/tmp/aspiresync.log
-#LOG_FILE=/dev/null
-#LOG_LEVEL=debug
-
 DOWNLOADS_FILESYSTEM=local
 #DOWNLOADS_DIR=/path/to/downloads/dir
 
@@ -32,6 +26,10 @@ DOWNLOADS_FILESYSTEM=local
 #S3_REGION=
 #S3_ENDPOINT=
 
+# in .env.local, use http://api.aspiredev.org/admin/api   (note http and not https)
+ASPIRECLOUD_ADMIN_API_URL=https://api.aspirecloud.io/admin/api
+ASPIRECLOUD_ADMIN_API_KEY=
+
 ###> symfony/framework-bundle ###
 APP_ENV=dev
 APP_SECRET=

diff --git a/README.md b/README.md
@@ -6,7 +6,7 @@ AspireSync is designed to enumerate and download the plugins and themes stored i
 
 * Download themes and plugins from the WordPress .org repository to the local filesystem or S3 storage.
 * Stores metadata and other information about every version of every plugin in sqlite.
-* Can export metadata as newline-delimited json for consumption by [AspireCloud](https://github.com/aspiresync/AspireCloud). 
+* Can export metadata as newline-delimited json for consumption by [AspireCloud](https://github.com/aspiresync/AspireCloud).
 * Can download all versions of plugins and themes, or just the latest version.
 * Handles closed and not found plugins/themes, preventing further download attempts for them.
 * Incremental updates, syncing only those items that have updated in subversion since the last sync.
@@ -34,3 +34,16 @@ You can configure the following environment variables to determine where uploads
 | S3_ENDPOINT          | The S3 API endpoint, only required if using S3 storage from a provider other than AWS                                    |
 | S3_KEY               | The AWS access_key_id for S3.  Optional if host/container roles are in effect.                                           |
 | S3_SECRET            | The companion secret to the `S3_KEY`;   Optional if host/container roles are in effect.                                  |
+
+### AspireCloud Integration
+
+To upload metadata to AspireCloud, use the
+`meta/bin/push-to-aspirecloud` script, which requires two environment variables:
+*
+*NOTE:
+** these variables must be set in your actual environment: putting them only in a .env file will not work!
+
+| Env Variable              | Description                                                                                            |
+|---------------------------|--------------------------------------------------------------------------------------------------------|
+| ASPIRECLOUD_ADMIN_API_URL | Base URL of AC admin API, e.g. `http://aspiredev.org/admin/api/v1`.  Do not include a trailing slash.  |
+| ASPIRECLOUD_ADMIN_API_KEY | API key generated in the AC instance.  Must belong to a user with RepoAdmin or SuperAdmin permissions. |
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,6 +4,9 @@ services:
       context: .
       dockerfile: docker/webapp/Dockerfile
       target: dev
+    environment:
+      ASPIRECLOUD_ADMIN_API_URL: ${ASPIRECLOUD_ADMIN_API_URL}
+      ASPIRECLOUD_ADMIN_API_KEY: ${ASPIRECLOUD_ADMIN_API_KEY}
     ports:
       - ${LOCAL_HTTP_PORT:-8199}:80
     volumes:

diff --git a/meta/bin/push-to-aspirecloud b/meta/bin/push-to-aspirecloud
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+#
+# EXAMPLE: bin/console sync:meta:dump:plugins --after='-1 hour' | meta/bin/push-to-aspirecloud
+#
+
+. $(dirname $0)/prelude.bash
+
+URL=${ASPIRECLOUD_ADMIN_API_URL:?variable not defined}
+KEY=${ASPIRECLOUD_ADMIN_API_KEY:?variable not defined}
+
+CHUNK_SIZE=${CHUNK_SIZE:-100} # values > 100 are likely to run into POST size limits
+
+CURL_ARGS=${CURL_ARGS:-'--compressed'}
+
+function main() {
+  local buffer=$(mktemp)
+  trap 'rm -f "$buffer"' EXIT
+
+  local count=0
+
+  while IFS= read -r line || [[ -n $line ]]; do
+    echo "$line" >> $buffer
+    (( count += 1 ))
+    echo -e -n "\ruploading $count ... "
+    [[ $(wc -l < $buffer) -ge $CHUNK_SIZE ]] && upload $buffer
+  done
+
+  upload $buffer
+
+  echo "done"
+}
+
+function upload() {
+  buffer=$1
+  [[ -s $buffer ]] && curl --silent -XPOST \
+    -H "Authorization: Bearer $KEY" \
+    -H 'Content-Type: application/nljson' \
+    -H 'Accept: application/json' \
+    --data-binary @"$buffer" \
+    $CURL_ARGS \
+    "$URL/v1/import" > /dev/null
+  cp /dev/null $buffer
+}
+
+main "$@"
diff --git a/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php b/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php
@@ -26,6 +26,8 @@ abstract class AbstractMetaFetchCommand extends AbstractBaseCommand
 {
     public const MAX_CONCURRENT_REQUESTS = 10;
 
+    private bool $clobber = false;
+
     public function __construct(
         protected readonly ListServiceInterface $listService,
         protected readonly MetadataServiceInterface $meta,
@@ -69,6 +71,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $this->log->notice("Running command {$this->getName()}");
         $this->startTimer();
 
+        $this->clobber = (bool) $input->getOption('update-all');
+
         $requested = array_fill_keys(StringUtil::explodeAndTrim($input->getOption($category) ?? ''), []);
         $min_age = (int) $input->getOption('skip-newer-than-secs');
 
@@ -124,6 +128,7 @@ protected function generateRequests(iterable $slugs): Generator
     protected function onResponse(Response $saloonResponse): void
     {
         $slug = null;
+
         try {
             $response = $saloonResponse->getPsrResponse();
             $request  = $saloonResponse->getRequest();
@@ -145,7 +150,7 @@ protected function onResponse(Response $saloonResponse): void
             } else {
                 $this->log->info("$slug ... No versions found");
             }
-            $this->meta->save($metadata);
+            $this->meta->save($metadata, $this->clobber);
         } catch (Exception $e) {
             $this->log->error("$slug ... ERROR: {$e->getMessage()}");
             return;

diff --git a/src/Commands/Sync/Meta/MetaDumpPluginsCommand.php b/src/Commands/Sync/Meta/MetaDumpPluginsCommand.php
@@ -8,6 +8,7 @@
 use App\Services\Metadata\PluginMetadataService;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class MetaDumpPluginsCommand extends AbstractBaseCommand
@@ -21,14 +22,18 @@ public function __construct(
     protected function configure(): void
     {
         $this->setName('sync:meta:dump:plugins')
-            ->setDescription('Dumps metadata of all plugins in jsonl format');
+            ->setDescription('Dumps metadata of all plugins in jsonl format')
+            ->addOption('after', null, InputOption::VALUE_REQUIRED, 'Dump only plugins synced after this date');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $this->startTimer();
 
-        foreach ($this->meta->exportAllMetadata() as $json) {
+        $after = $input->getOption('after');
+        $timestamp = $after ? \Safe\strtotime($after) : 0;
+
+        foreach ($this->meta->exportAllMetadata($timestamp) as $json) {
             echo $json . PHP_EOL;
         }
 

diff --git a/src/Commands/Sync/Meta/MetaDumpThemesCommand.php b/src/Commands/Sync/Meta/MetaDumpThemesCommand.php
@@ -8,6 +8,7 @@
 use App\Services\Metadata\ThemeMetadataService;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class MetaDumpThemesCommand extends AbstractBaseCommand
@@ -21,14 +22,18 @@ public function __construct(
     protected function configure(): void
     {
         $this->setName('sync:meta:dump:themes')
-            ->setDescription('Dumps metadata of all themes in jsonl format');
+            ->setDescription('Dumps metadata of all themes in jsonl format')
+            ->addOption('after', null, InputOption::VALUE_REQUIRED, 'Dump only plugins synced after this date');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $this->startTimer();
 
-        foreach ($this->meta->exportAllMetadata() as $json) {
+        $after = $input->getOption('after');
+        $timestamp = $after ? \Safe\strtotime($after) : 0;
+
+        foreach ($this->meta->exportAllMetadata($timestamp) as $json) {
             echo $json . PHP_EOL;
         }