[pull] main from activepieces:main

.github/workflows/release-pieces.yml

@@ -59,36 +59,63 @@ jobs:
         run: cp .npmrc $HOME/.npmrc
 
       - name: publish shared package
-        run: npx turbo run build --filter=@activepieces/shared && npx ts-node -r tsconfig-paths/register -P packages/engine/tsconfig.lib.json tools/scripts/utils/publish-nx-project.ts packages/shared
+        run: npx turbo run build --filter=@activepieces/shared --force && npx ts-node -r tsconfig-paths/register -P packages/engine/tsconfig.lib.json tools/scripts/utils/publish-npm-package.ts packages/shared 
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
       - name: publish pieces-common package
-        run: npx turbo run build --filter=@activepieces/pieces-common && npx ts-node -r tsconfig-paths/register -P packages/engine/tsconfig.lib.json tools/scripts/utils/publish-nx-project.ts packages/pieces/community/common
+        run: npx turbo run build --filter=@activepieces/pieces-common --force && npx ts-node -r tsconfig-paths/register -P packages/engine/tsconfig.lib.json tools/scripts/utils/publish-npm-package.ts packages/pieces/community/common
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
       - name: publish pieces-framework package
-        run: npx turbo run build --filter=@activepieces/pieces-framework && npx ts-node -r tsconfig-paths/register -P packages/engine/tsconfig.lib.json tools/scripts/utils/publish-nx-project.ts packages/pieces/community/framework
+        run: npx turbo run build --filter=@activepieces/pieces-framework --force && npx ts-node -r tsconfig-paths/register -P packages/engine/tsconfig.lib.json tools/scripts/utils/publish-npm-package.ts packages/pieces/community/framework
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
-      - name: build packages
-        run: npx turbo run build
+      - name: Find changed pieces
+        id: changed
+        run: |
+          OUTPUT=$(npx ts-node -r tsconfig-paths/register -P packages/engine/tsconfig.lib.json tools/scripts/pieces/find-changed-pieces.ts)
+
+          TURBO_FILTER=$(echo "$OUTPUT" | grep '^TURBO_FILTER:' | sed 's/^TURBO_FILTER://')
+          CHANGED_DIRS=$(echo "$OUTPUT" | sed -n '/^CHANGED_DIRS:$/,/^TURBO_FILTER:/{ /^CHANGED_DIRS:$/d; /^TURBO_FILTER:/d; p; }')
+
+          if [ -z "$TURBO_FILTER" ]; then
+            echo "No changed pieces found"
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+            echo "turbo_filter=$TURBO_FILTER" >> "$GITHUB_OUTPUT"
+            {
+              echo "changed_dirs<<PIECES_EOF"
+              echo "$CHANGED_DIRS"
+              echo "PIECES_EOF"
+            } >> "$GITHUB_OUTPUT"
+            echo "Changed pieces: $TURBO_FILTER"
+          fi
+
+      - name: Build changed pieces
+        if: steps.changed.outputs.has_changes == 'true'
+        run: npx turbo run build ${{ steps.changed.outputs.turbo_filter }} --force
 
       - name: publish pieces packages
+        if: steps.changed.outputs.has_changes == 'true'
         run: npx ts-node -r tsconfig-paths/register -P packages/engine/tsconfig.lib.json tools/scripts/pieces/publish-pieces-to-npm.ts
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          CHANGED_PIECES: ${{ steps.changed.outputs.changed_dirs }}
 
       - name: Check available memory
         run: free -h
 
       - name: update pieces metadata
+        if: steps.changed.outputs.has_changes == 'true'
         run: npx ts-node -r tsconfig-paths/register -r ./tools/scripts/fix-dts-require.js -P packages/engine/tsconfig.lib.json tools/scripts/pieces/update-pieces-metadata.ts packages/pieces/community/framework
         env:
           AP_CLOUD_API_KEY: ${{ secrets.AP_CLOUD_API_KEY }}
           NODE_OPTIONS: "--max-old-space-size=8192"
+          CHANGED_PIECES: ${{ steps.changed.outputs.changed_dirs }}
 
       - name: Notify Discord on failure
         if: failure()

tools/scripts/pieces/find-changed-pieces.ts

@@ -0,0 +1,65 @@
+import { readPackageJson } from '../utils/files'
+import { findAllPiecesDirectoryInSource, NON_PIECES_PACKAGES, AP_CLOUD_API_BASE } from '../utils/piece-script-utils'
+
+const main = async () => {
+  const release = (await readPackageJson('.')).version
+  console.info(`[findChangedPieces] release=${release}`)
+
+  const registry = await fetchRegistry(release)
+  console.info(`[findChangedPieces] registry has ${registry.size} name@version entries`)
+
+  const allPieceDirs = await findAllPiecesDirectoryInSource()
+  const changedDirs: string[] = []
+  const turboFilters: string[] = []
+
+  for (const dir of allPieceDirs) {
+    const pkg = await readPackageJson(dir)
+    if (NON_PIECES_PACKAGES.includes(pkg.name)) {
+      continue
+    }
+    const key = `${pkg.name}@${pkg.version}`
+    if (!registry.has(key)) {
+      changedDirs.push(dir)
+      turboFilters.push(`--filter=${pkg.name}`)
+      console.info(`[findChangedPieces] changed: ${key} (${dir})`)
+    }
+  }
+
+  console.info(`[findChangedPieces] ${changedDirs.length} pieces not on cloud`)
+
+  // Output for GitHub Actions
+  const output = {
+    count: changedDirs.length,
+    dirs: changedDirs,
+    turboFilter: turboFilters.join(' '),
+  }
+  // Print as JSON so the workflow can parse it
+  console.log(`::set-output-json::${JSON.stringify(output)}`)
+  // Also print dirs one per line for GITHUB_OUTPUT
+  if (changedDirs.length > 0) {
+    console.log(`CHANGED_DIRS:\n${changedDirs.join('\n')}`)
+    console.log(`TURBO_FILTER:${turboFilters.join(' ')}`)
+  }
+}
+
+async function fetchRegistry(release: string): Promise<Set<string>> {
+  const url = `${AP_CLOUD_API_BASE}/pieces/registry?release=${release}&edition=ee`
+  console.info(`[fetchRegistry] fetching ${url}`)
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`Failed to fetch registry: ${response.status} ${response.statusText}`)
+  }
+  const entries: RegistryEntry[] = await response.json()
+  const set = new Set<string>()
+  for (const entry of entries) {
+    set.add(`${entry.name}@${entry.version}`)
+  }
+  return set
+}
+
+main()
+
+type RegistryEntry = {
+  name: string
+  version: string
+}

tools/scripts/pieces/publish-piece.ts

@@ -6,7 +6,7 @@ import { findAllPiecesDirectoryInSource } from '../utils/piece-script-utils'
 import { isNil } from '@activepieces/shared'
 import chalk from 'chalk'
 import path from 'node:path'
-import { publishNxProject } from '../utils/publish-nx-project'
+import { publishNpmPackage } from '../utils/publish-npm-package'
 
 export const publishPiece = async (name: string): Promise<void> => {
   assert(name, '[publishPiece] parameter "name" is required')
@@ -20,11 +20,10 @@ export const publishPiece = async (name: string): Promise<void> => {
 
   await exec(`turbo run build --filter=./packages/pieces/community/${name}`)
 
-  await publishNxProject(directory)
+  await publishNpmPackage(directory)
 
   const { version } = await readPackageJson(directory)
   console.info(chalk.green.bold(`[publishPiece] success, name=${name}, version=${version}`))
-
 }
 
 const main = async (): Promise<void> => {

tools/scripts/pieces/publish-pieces-to-npm.ts

@@ -1,18 +1,25 @@
-import { publishNxProject } from '../utils/publish-nx-project'
+import { publishNpmPackage } from '../utils/publish-npm-package'
 import { findAllPiecesDirectoryInSource } from '../utils/piece-script-utils'
 import { chunk } from '../../../packages/shared/src/lib/common/utils/utils'
 
-const publishPiece = async (nxProjectPath: string): Promise<void> => {
-  console.info(`[publishPiece] nxProjectPath=${nxProjectPath}`)
-  await publishNxProject(nxProjectPath)
+function getChangedPiecePaths(): string[] | null {
+  const changedPieces = process.env['CHANGED_PIECES']
+  if (!changedPieces || changedPieces.trim() === '') {
+    return null
+  }
+  return changedPieces.split('\n').filter(Boolean)
 }
 
 const main = async () => {
-  const piecesSource = await findAllPiecesDirectoryInSource()
+  const changedPaths = getChangedPiecePaths()
+  const piecesSource = changedPaths ?? await findAllPiecesDirectoryInSource()
+
+  console.info(`[publishPieces] publishing ${piecesSource.length} pieces${changedPaths ? ' (scoped to changed)' : ' (all)'}`)
+
   const piecesSourceChunks = chunk(piecesSource, 30)
 
   for (const chunk of piecesSourceChunks) {
-    await Promise.all(chunk.map(publishPiece))
+    await Promise.all(chunk.map((path) => publishNpmPackage(path)))
     await new Promise(resolve => setTimeout(resolve, 5000))
   }
 }

tools/scripts/utils/piece-script-utils.ts

@@ -1,6 +1,7 @@
 
+import { existsSync } from 'node:fs'
 import { readdir, stat } from 'node:fs/promises'
-import { resolve, join } from 'node:path'
+import { resolve, join, relative } from 'node:path'
 import { cwd } from 'node:process'
 import { extractPieceFromModule } from '@activepieces/shared'
 import * as semver from 'semver'
@@ -59,7 +60,7 @@ export function getCommunityPieceFolder(pieceName: string): string {
 export async function findAllPiecesDirectoryInSource(): Promise<string[]> {
     const piecesPath = resolve(cwd(), 'packages', 'pieces')
     const paths = await traverseFolder(piecesPath)
-    return paths
+    return paths.map(p => relative(cwd(), p))
 }
 
 export const pieceMetadataExists = async (
@@ -86,9 +87,13 @@ export const pieceMetadataExists = async (
 };
 
 export async function findNewPieces(): Promise<PieceMetadata[]> {
-    const paths = await findAllDistPaths()
+    const changedDistPaths = getChangedPiecesDistPaths()
+    const paths = changedDistPaths ?? await findAllDistPaths()
+
+    console.info(`[findNewPieces] scanning ${paths.length} dist paths${changedDistPaths ? ' (scoped to changed)' : ' (all)'}`)
+
     const changedPieces: PieceMetadata[] = []
-    
+
     // Adding batches because of memory limit when we have a lot of pieces
     const batchSize = 75
     for (let i = 0; i < paths.length; i += batchSize) {
@@ -108,14 +113,30 @@ export async function findNewPieces(): Promise<PieceMetadata[]> {
             }
             return null;
         }))
-        
+
         const validResults = batchResults.filter((piece): piece is PieceMetadata => piece !== null)
         changedPieces.push(...validResults)
     }
-    
+
     return changedPieces;
 }
 
+function getChangedPiecesDistPaths(): string[] | null {
+    const changedPieces = process.env['CHANGED_PIECES']
+    if (!changedPieces || changedPieces.trim() === '') {
+        return null
+    }
+    return changedPieces.split('\n').filter(Boolean).map(p => {
+        return resolve(cwd(), 'dist', p)
+    }).filter(p => {
+        const exists = existsSync(join(p, 'package.json'))
+        if (!exists) {
+            console.info(`[getChangedPiecesDistPaths] skipping, no build output at ${p}`)
+        }
+        return exists
+    })
+}
+
 export async function findAllPieces(): Promise<PieceMetadata[]> {
     const paths = await findAllDistPaths()
     const pieces = await Promise.all(paths.map((p) => loadPieceFromFolder(p)))

tools/scripts/utils/publish-nx-project.ts → tools/scripts/utils/publish-npm-package.ts

@@ -1,33 +1,32 @@
 import assert from 'node:assert'
 import { argv } from 'node:process'
 import { execSync } from 'node:child_process'
-import { readFileSync, writeFileSync } from 'node:fs'
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
 import { readPackageJson } from './files'
 import { packagePrePublishChecks } from './package-pre-publish-checks'
 
-export const publishNxProject = async (path: string): Promise<void> => {
-  console.info(`[publishProject] path=${path}`)
-  assert(path, '[publishProject] parameter "path" is required')
+export const publishNpmPackage = async (path: string): Promise<void> => {
+  console.info(`[publishPackage] path=${path}`)
+  assert(path, '[publishPackage] parameter "path" is required')
 
-  const packageAlreadyPublished = await packagePrePublishChecks(path);
+  // Output path follows the convention: dist/{source-path}
+  const outputPath = `dist/${path}`
 
+  if (!existsSync(`${outputPath}/package.json`)) {
+    console.info(`[publishPackage] skipping, no build output at ${outputPath}`)
+    return
+  }
+
+  const packageAlreadyPublished = await packagePrePublishChecks(path);
   if (packageAlreadyPublished) {
     return;
   }
-
   const { version } = await readPackageJson(path)
 
-  // Output path follows the convention: dist/{source-path}
-  const outputPath = `dist/${path}`
-
   // Update version in dist package.json before publishing
-  try {
-    const json = JSON.parse(readFileSync(`${outputPath}/package.json`).toString())
-    json.version = version
-    writeFileSync(`${outputPath}/package.json`, JSON.stringify(json, null, 2))
-  } catch (e) {
-    console.error(`Error reading package.json file from build output at ${outputPath}`)
-  }
+  const json = JSON.parse(readFileSync(`${outputPath}/package.json`).toString())
+  json.version = version
+  writeFileSync(`${outputPath}/package.json`, JSON.stringify(json, null, 2))
 
   execSync(`npm publish --access public --tag latest`, { cwd: outputPath, stdio: 'inherit' })
 
@@ -36,7 +35,7 @@ export const publishNxProject = async (path: string): Promise<void> => {
 
 const main = async (): Promise<void> => {
   const path = argv[2]
-  await publishNxProject(path)
+  await publishNpmPackage(path)
 }
 
 /*

turbo.json

@@ -11,7 +11,8 @@
         "!src/**/*.spec.ts",
         "!src/**/*.test.ts",
         "!vitest.config.*"
-      ]
+      ],
+      "cache": false
     },
     "react-ui#build": {
       "dependsOn": ["^build"],
@@ -52,6 +53,7 @@
       ]
     },
     "lint": {
+      "dependsOn": ["^build"],
       "inputs": ["src/**", ".eslintrc.json", "../../.eslintrc.json"],
       "outputs": []
     },