merged master

Signed-off-by: Kevin Su <pingsutw@apache.org>

.github/actions/clear-action-cache/action.yml

@@ -0,0 +1,11 @@
+name: 'Clear action cache'
+description: 'As suggested by GitHub to prevent low disk space: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173'
+runs:
+  using: 'composite'
+  steps:
+    - shell: bash
+      run: |
+        rm -rf /usr/share/dotnet
+        rm -rf /opt/ghc
+        rm -rf "/usr/local/share/boost"
+        rm -rf "$AGENT_TOOLSDIRECTORY"

.github/workflows/create_release.yml

@@ -28,7 +28,6 @@ jobs:
               "datacatalog",
               "flyteadmin",
               "flytecopilot",
-              "flyteidl",
               "flyteplugins",
               "flytepropeller",
               "flytestdlib",

.github/workflows/flytectl-release.yml

@@ -1,13 +1,34 @@
 name: Flytectl release
 
 on:
-  push:
-    tags:
-      - flytectl/v*.*.*
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "version. Do *not* use the `flytectl/` prefix, e.g. `flytectl/v1.2.3`, instead use only `v1.2.3` (including the `v`)"
+        required: true
 
 jobs:
+  push-flytectl-tag:
+    name: Push git tag containing the `flyteidl/` prefix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.FLYTE_BOT_PAT }}
+          script: |
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/tags/flytectl/${{ github.event.inputs.version }}`,
+              sha: context.sha
+            })
   release:
     name: Goreleaser
+    needs:
+      - push-flytectl-tag
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.github/workflows/flyteidl-release.yml

@@ -1,18 +1,41 @@
-name: Upload flyteidl to PyPI and npm
+name: Release flyteidl
 
 on:
-  push:
-    tags:
-      - flyteidl/v*.*.*
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "version. Do *not* use the `flyteidl/` prefix, e.g. `flyteidl/v1.2.3`, instead use only `v1.2.3` (including the `v`)"
+        required: true
 
 jobs:
+  push-flyteidl-tag:
+    name: Push git tag containing the `flyteidl/` prefix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.FLYTE_BOT_PAT }}
+          script: |
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/tags/flyteidl/${{ github.event.inputs.version }}`,
+              sha: context.sha
+            })
   deploy-to-pypi:
+    needs:
+      - push-flyteidl-tag
     runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: flyteidl
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
@@ -29,6 +52,8 @@ jobs:
           python -m build
           twine upload dist/*
   deploy-to-npm:
+    needs:
+      - push-flyteidl-tag
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -41,8 +66,8 @@ jobs:
           registry-url: "https://registry.npmjs.org"
       - name: Set version in npm package
         run: |
-          # from refs/tags/v1.2.3 get 1.2.3
-          VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+          # v1.2.3 get 1.2.3
+          VERSION=$(echo ${{ inputs.version }} | sed 's#.*v##')
           VERSION=$VERSION make update_npmversion
         shell: bash
       - run: |

.github/workflows/single-binary.yml

@@ -175,7 +175,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install uv
-          uv pip install --system flytekit flytekitplugins-deck-standard flytekitplugins-envd "numpy<2.0.0"
+          uv pip install --system flytekit flytekitplugins-deck-standard flytekitplugins-envd "numpy<2.0.0" pyarrow
           uv pip freeze
       - name: Checkout flytesnacks
         uses: actions/checkout@v4

.github/workflows/tests.yml

@@ -30,12 +30,14 @@ jobs:
       - name: Fetch flyte code
         uses: actions/checkout@v4
         with:
-          path: "${{ github.workspace }}/flyte"
+          path: flyte
+      - name: 'Clear action cache'
+        uses: ./flyte/.github/actions/clear-action-cache
       - name: Fetch flytekit code
         uses: actions/checkout@v4
         with:
           repository: flyteorg/flytekit
-          path: "${{ github.workspace }}/flytekit"
+          path: flytekit
       - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true

CHANGELOG/CHANGELOG-v0.2.0.md

@@ -16,7 +16,7 @@
 - RawOutputDirectories created in FlytePropeller
 - Improve visibility and observability
 - User/System error differentiation
-- Optional interruptible tasks (lets use spot instances, reduce cost)
+- Optional interruptible tasks (let's use spot instances, to reduce cost)
 - Caps on queue time for workflows
 - Multi cluster improvements
 - Visibility into execution cluster for the execution

CHANGELOG/CHANGELOG-v1.0.2-b1.md

@@ -5,7 +5,7 @@
 1. [Bugfix](https://github.com/flyteorg/flyte/issues/2444) With GRPC v1.46.0 non-ascii chars are not permitted in grpc metadata
 1. [Housekeeping](https://github.com/flyteorg/flyte/issues/1698) Configure grpc_health_prob in admin
 1. [Feature](https://github.com/flyteorg/flyte/issues/2329) In Flytectl use Launchplan with latest version for scheduled workflows
-1. [Bugfix](https://github.com/flyteorg/flyte/issues/2262) Pods started before InjectFinalizer is disabled are never deleted
+1. [Bugfix](https://github.com/flyteorg/flyte/issues/2262) Pods started before InjectFinalizer was disabled are never deleted
 1. [Housekeeping](https://github.com/flyteorg/flyte/issues/2504) Checksum grpc_health_probe
 1. [Feature](https://github.com/flyteorg/flyte/issues/2284) Allow to choose Spot Instances at workflow start time
 1. [Feature](https://github.com/flyteorg/flyte/pull/2439) Use the same pod annotation formatting in syncresources cronjob

CHANGELOG/CHANGELOG-v1.1.0.md

@@ -17,7 +17,7 @@ Support for [Optional types](https://github.com/flyteorg/flyte/issues/2426). Wit
 
 ### Bug Fixes
 * [Propeller](https://github.com/flyteorg/flyte/issues/2298) calling finalize rather than abort
-* [Propeller](https://github.com/flyteorg/flyte/issues/2404) correctly identify error when requesting a launch plan that does not exist.
+* [Propeller](https://github.com/flyteorg/flyte/issues/2404) correctly identifies an error when requesting a launch plan that does not exist.
 * Better handle [execution CRDs](https://github.com/flyteorg/flyte/issues/2275) that don't exist in Admin.
 * [Fix panic](https://github.com/flyteorg/flyte/issues/2597) when creating additional label options.
 * Check [validity](https://github.com/flyteorg/flyte/issues/2601) of notifications.

CHANGELOG/CHANGELOG-v1.13.1-rc0.md

@@ -0,0 +1,65 @@
+# Flyte v1.13.1-rc0 Release Notes
+
+## Changelog
+* chore: update runllm widget configuration by @agiron123 in https://github.com/flyteorg/flyte/pull/5530
+* Use jsonpb AllowUnknownFields everywhere by @andrewwdye in https://github.com/flyteorg/flyte/pull/5521
+* [flyteadmin] Use WithContext in all DB calls by @Sovietaced in https://github.com/flyteorg/flyte/pull/5538
+* add new project state: SYSTEM_ARCHIVED by @troychiu in https://github.com/flyteorg/flyte/pull/5544
+* Pryce/doc 434 clarify how code is pushed into a given image during pyflyte by @pryce-turner in https://github.com/flyteorg/flyte/pull/5548
+* Increase more memory limits in flyteagent by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5550
+* Updated map task information to indicate array node is now the default and optional return type by @pryce-turner in https://github.com/flyteorg/flyte/pull/5561
+* reverted mockery-v2 on ExecutionContext by @hamersaw in https://github.com/flyteorg/flyte/pull/5562
+* Fix issues with helm chart release process by @Sovietaced in https://github.com/flyteorg/flyte/pull/5560
+* Add FlyteDirectory to file_types.rst template by @ppiegaze in https://github.com/flyteorg/flyte/pull/5564
+* Fully populate Abort task event fields by @va6996 in https://github.com/flyteorg/flyte/pull/5551
+* [fix] Add blob typechecker by @ddl-rliu in https://github.com/flyteorg/flyte/pull/5519
+* Refactor echo plugin by @pingsutw in https://github.com/flyteorg/flyte/pull/5565
+* [Bug] fix ArrayNode state's TaskPhase reset by @pvditt in https://github.com/flyteorg/flyte/pull/5451
+* Remove confusing prometheus configuration options in helm charts  by @Sovietaced in https://github.com/flyteorg/flyte/pull/5549
+* [Housekeeping] Bump Go version to 1.22 by @lowc1012 in https://github.com/flyteorg/flyte/pull/5032
+* Fix typos by @omahs in https://github.com/flyteorg/flyte/pull/5571
+* Respect original task definition retry strategy for single task executions by @katrogan in https://github.com/flyteorg/flyte/pull/5577
+* Clarify the support for the Java/Scala SDK in the docs by @eapolinario in https://github.com/flyteorg/flyte/pull/5582
+* Fix spelling issues by @nnsW3 in https://github.com/flyteorg/flyte/pull/5580
+* Give flyte binary cluster role permission to create service accounts by @shreyas44 in https://github.com/flyteorg/flyte/pull/5579
+* Simplify single task retry strategy check by @eapolinario in https://github.com/flyteorg/flyte/pull/5584
+* Fix failures in `generate_helm` CI check by @eapolinario in https://github.com/flyteorg/flyte/pull/5587
+* Update GPU docs by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/5515
+* Update azblob 1.1.0 -> 1.4.0 / azcore 1.7.2 -> 1.13.0 by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5590
+* Bump github.com/go-jose/go-jose/v3 from 3.0.0 to 3.0.3 in /flyteadmin by @dependabot in https://github.com/flyteorg/flyte/pull/5591
+* add execution mode to ArrayNode proto by @pvditt in https://github.com/flyteorg/flyte/pull/5512
+* Fix incorrect YAML for unpartitoned GPU by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/5595
+* Another YAML fix by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/5596
+* DOC-431 Document pyflyte option --overwrite-cache by @ppiegaze in https://github.com/flyteorg/flyte/pull/5567
+* Upgrade docker dependency to address vulnerability by @katrogan in https://github.com/flyteorg/flyte/pull/5614
+* Support offloading workflow CRD inputs by @katrogan in https://github.com/flyteorg/flyte/pull/5609
+* [flyteadmin] Refactor panic recovery into middleware by @Sovietaced in https://github.com/flyteorg/flyte/pull/5546
+* Snowflake agent Doc by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5620
+* [flytepropeller][compiler] Error Handling when Type is not found by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5612
+* Fix nil pointer when task plugin load returns error by @Sovietaced in https://github.com/flyteorg/flyte/pull/5622
+* Log stack trace when refresh cache sync recovers from panic by @Sovietaced in https://github.com/flyteorg/flyte/pull/5623
+* [Doc] Fix snowflake agent secret documentation error by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5626
+* [Doc] Explain how Agent Secret Works by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5625
+* Fix typo in execution manager by @ddl-rliu in https://github.com/flyteorg/flyte/pull/5619
+* Amend Admin to use grpc message size by @wild-endeavor in https://github.com/flyteorg/flyte/pull/5628
+* [Docs] document the process of setting ttl for a ray cluster by @pingsutw in https://github.com/flyteorg/flyte/pull/5636
+* Add CustomHeaderMatcher to pass additional headers by @andrewwdye in https://github.com/flyteorg/flyte/pull/5563
+* Turn flyteidl and flytectl releases into manual gh workflows by @eapolinario in https://github.com/flyteorg/flyte/pull/5635
+* docs: fix typo by @cratiu222 in https://github.com/flyteorg/flyte/pull/5643
+* Use enable_deck=True in docs by @thomasjpfan in https://github.com/flyteorg/flyte/pull/5645
+* Fix flyteidl release  checkout all tags by @eapolinario in https://github.com/flyteorg/flyte/pull/5646
+* Install pyarrow in sandbox functional tests by @eapolinario in https://github.com/flyteorg/flyte/pull/5647
+* docs: add documentation for configuring notifications in GCP by @desihsu in https://github.com/flyteorg/flyte/pull/5545
+* Correct "sucessfile" to "successfile" by @shengyu7697 in https://github.com/flyteorg/flyte/pull/5652
+* Fix ordering for custom template values in cluster resource controller by @katrogan in https://github.com/flyteorg/flyte/pull/5648
+* Don't error when attempting to trigger schedules for inactive projects by @katrogan in https://github.com/flyteorg/flyte/pull/5649
+
+## New Contributors
+* @omahs made their first contribution in https://github.com/flyteorg/flyte/pull/5571
+* @nnsW3 made their first contribution in https://github.com/flyteorg/flyte/pull/5580
+* @shreyas44 made their first contribution in https://github.com/flyteorg/flyte/pull/5579
+* @cratiu222 made their first contribution in https://github.com/flyteorg/flyte/pull/5643
+* @desihsu made their first contribution in https://github.com/flyteorg/flyte/pull/5545
+* @shengyu7697 made their first contribution in https://github.com/flyteorg/flyte/pull/5652
+
+**Full Changelog**: https://github.com/flyteorg/flyte/compare/v1.13.0...v1.13.1-rc0

CHANGELOG/CHANGELOG-v1.3.0.md

@@ -7,7 +7,7 @@ The main features of this 1.3 release are
 * Signaling/gate node support (human in the loop tasks)
 * User documentation support (backend and flytekit only, limited types)
 
-The latter two are pending some work in Flyte console, they will be piped through fully by the end of Q1. Support for setting and approving gate nodes is supported in `FlyteRemote` however, though only a limited set of types can be passed in.
+The latter two are pending some work in Flyte console, they will be piped through fully by the end of Q1. Support for setting and approving gate nodes is supported in `FlyteRemote` however, only a limited set of types can be passed in.
 
 ## Notes
 There are a couple things to point out with this release.

charts/flyte-binary/README.md

@@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment
 | configuration.auth.oidc.clientId | string | `""` |  |
 | configuration.auth.oidc.clientSecret | string | `""` |  |
 | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` |  |
-| configuration.co-pilot.image.tag | string | `"v1.13.0"` |  |
+| configuration.co-pilot.image.tag | string | `"v1.13.1-rc0"` |  |
 | configuration.database.dbname | string | `"flyte"` |  |
 | configuration.database.host | string | `"127.0.0.1"` |  |
 | configuration.database.options | string | `"sslmode=disable"` |  |

charts/flyte-binary/values.yaml

@@ -159,7 +159,7 @@ configuration:
       # repository CoPilot sidecar image repository
       repository: cr.flyte.org/flyteorg/flytecopilot  # FLYTECOPILOT_IMAGE
       # tag CoPilot sidecar image tag
-      tag: v1.13.0  # FLYTECOPILOT_TAG
+      tag: v1.13.1-rc0  # FLYTECOPILOT_TAG
   # agentService Flyte Agent configuration
   agentService:
     defaultAgent: