resolve conflict

Signed-off-by: zerafachris PERSONAL <zerafachris@gmail.com>
zerafachris · Jan 6, 2025 · 897df59 · 897df59
2 parents 1ef5d0a + 07958f7
commit 897df59
Show file tree

Hide file tree

Showing 130 changed files with 7,841 additions and 2,592 deletions.
diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml
@@ -72,13 +72,13 @@ jobs:
           java-version: '11'
           java-package: jdk
           architecture: x64
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-it-maven-
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }}

diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml
@@ -53,13 +53,13 @@ jobs:
           java-version: '11'
           java-package: jdk
           architecture: x64
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-it-maven-
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }}
@@ -97,11 +97,11 @@ jobs:
           python-version: "3.11"
           architecture: x64
       - name: Authenticate to Google Cloud
-        uses: 'google-github-actions/auth@v1'
+        uses: google-github-actions/auth@v2
         with:
           credentials_json: '${{ secrets.GCP_SA_KEY }}'
       - name: Set up gcloud SDK
-        uses: google-github-actions/setup-gcloud@v1
+        uses: google-github-actions/setup-gcloud@v2
         with:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
       - run: gcloud auth configure-docker --quiet
@@ -137,18 +137,18 @@ jobs:
         with:
           python-version: '3.11'
           architecture: 'x64'
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-it-maven-
       - name: Authenticate to Google Cloud
-        uses: 'google-github-actions/auth@v1'
+        uses: google-github-actions/auth@v2
         with:
           credentials_json: '${{ secrets.GCP_SA_KEY }}'
       - name: Set up gcloud SDK
-        uses: google-github-actions/setup-gcloud@v1
+        uses: google-github-actions/setup-gcloud@v2
         with:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
       - name: Use gcloud CLI

diff --git a/.github/workflows/lint_pr.yml b/.github/workflows/lint_pr.yml
@@ -14,7 +14,7 @@ jobs:
     name: Validate PR title
     runs-on: ubuntu-latest
     steps:
-      - uses: amannn/action-semantic-pull-request@v4
+      - uses: amannn/action-semantic-pull-request@v5
         with:
           # Must use uppercase
           subjectPattern: ^(?=[A-Z]).+$

diff --git a/.github/workflows/operator_pr.yml b/.github/workflows/operator_pr.yml
@@ -7,7 +7,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
           go-version: 1.21.x
       - name: Operator tests

diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml
@@ -45,12 +45,12 @@ jobs:
       - name: Get uv cache dir
         id: uv-cache
         run: |
-          echo "::set-output name=dir::$(uv cache dir)"
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
       - name: uv cache
         uses: actions/cache@v4
         with:
           path: ${{ steps.uv-cache.outputs.dir }}
-          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
+          key: ${{ runner.os }}-${{ matrix.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', matrix.python-version)) }}
       - name: Install dependencies
         run: make install-python-dependencies-ci
       - name: Test local integration tests

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -20,8 +20,8 @@ on:
         type: boolean
 
 jobs:
-  if: github.repository == 'feast-dev/feast'
   get_dry_release_versions:
+    if: github.repository == 'feast-dev/feast'
     runs-on: ubuntu-latest
     env:
       GITHUB_TOKEN: ${{ github.event.inputs.token }}

diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml
@@ -31,7 +31,7 @@ jobs:
       - name: Get uv cache dir
         id: uv-cache
         run: |
-          echo "::set-output name=dir::$(uv cache dir)"
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
       - name: uv cache
         uses: actions/cache@v4
         with:

diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -13,10 +13,15 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [ "3.9", "3.10", "3.11"]
-        os: [ ubuntu-latest, macos-13 ]
+        os: [ ubuntu-latest, macos-13, macos-14 ]
         exclude:
           - os: macos-13
             python-version: "3.9"
+          - os: macos-14
+            python-version: "3.9"
+          - os: macos-14
+            python-version: "3.10"
+
     env:
       OS: ${{ matrix.os }}
       PYTHON: ${{ matrix.python-version }}
@@ -33,8 +38,8 @@ jobs:
           curl -LsSf https://astral.sh/uv/install.sh | sh
       - name: Get uv cache dir
         id: uv-cache
-        run: |
-          echo "::set-output name=dir::$(uv cache dir)"
+        run: |          
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
       - name: uv cache
         uses: actions/cache@v4
         with:
@@ -52,7 +57,7 @@ jobs:
       NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: './ui/.nvmrc'
           registry-url: 'https://registry.npmjs.org'

diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@ scratch*
 
 ### Local Environment ###
 *local*.env
+tools
 
 ### Secret ###
 **/service_account.json
@@ -101,6 +102,7 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
+coverage.out
 *.cover
 .hypothesis/
 .pytest_cache/
@@ -222,4 +224,4 @@ ui/.vercel
 **/yarn-error.log*
 
 # Go subprocess binaries (built during feast pip package building)
-sdk/python/feast/binaries/
+sdk/python/feast/binaries/
diff --git a/Makefile b/Makefile
@@ -14,7 +14,13 @@
 #  limitations under the License.
 #
 
-ROOT_DIR 	:= $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+
+# install tools in project (tool) dir to not pollute the system
+TOOL_DIR := $(ROOT_DIR)/tools
+export GOBIN=$(TOOL_DIR)/bin
+export PATH := $(TOOL_DIR)/bin:$(PATH)
+
 MVN := mvn -f java/pom.xml ${MAVEN_EXTRA_OPTS}
 OS := linux
 ifeq ($(shell uname -s), Darwin)
@@ -24,12 +30,15 @@ TRINO_VERSION ?= 376
 PYTHON_VERSION = ${shell python --version | grep -Eo '[0-9]\.[0-9]+'}
 
 PYTHON_VERSIONS := 3.9 3.10 3.11
+
 define get_env_name
 $(subst .,,py$(1))
 endef
 
 
 # General
+$(TOOL_DIR):
+	mkdir -p $@/bin
 
 format: format-python format-java
 
@@ -96,14 +105,14 @@ test-python-unit:
 	python -m pytest -n 8 --color=yes sdk/python/tests
 
 test-python-integration:
-	python -m pytest -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
+	python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
 		-k "(not snowflake or not test_historical_features_main)" \
 		sdk/python/tests
 
 test-python-integration-local:
 	FEAST_IS_LOCAL_TEST=True \
 	FEAST_LOCAL_ONLINE_CONTAINER=True \
-	python -m pytest -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
+	python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
 		-k "not test_lambda_materialization and not test_snowflake_materialization" \
 		sdk/python/tests
 
@@ -268,7 +277,7 @@ test-python-universal-postgres-online:
 				not test_snowflake" \
  			sdk/python/tests
 
- test-python-universal-mysql-online:
+test-python-universal-mysql-online:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.mysql_online_store.mysql_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \
@@ -292,7 +301,11 @@ test-python-universal-cassandra:
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.cassandra_online_store.cassandra_repo_configuration \
 	PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \
 	python -m pytest -x --integration \
-	sdk/python/tests
+	sdk/python/tests/integration/offline_store/test_feature_logging.py \
+		--ignore=sdk/python/tests/integration/offline_store/test_validation.py \
+		-k "not test_snowflake and \
+			not test_spark_materialization_consistency and \
+			not test_universal_materialization"
 
 test-python-universal-hazelcast:
 	PYTHONPATH='.' \
@@ -330,7 +343,7 @@ test-python-universal-cassandra-no-cloud-providers:
 	  not test_snowflake" \
 	sdk/python/tests
 
- test-python-universal-elasticsearch-online:
+test-python-universal-elasticsearch-online:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.elasticsearch_online_store.elasticsearch_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.elasticsearch \
@@ -349,6 +362,14 @@ test-python-universal-cassandra-no-cloud-providers:
 				not test_snowflake" \
  			sdk/python/tests
 
+test-python-universal-milvus-online:
+	PYTHONPATH='.' \
+		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.milvus_online_store.milvus_repo_configuration \
+		PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.milvus \
+		python -m pytest -n 8 --integration \
+		-k "test_retrieve_online_milvus_documents" \
+ 			sdk/python/tests --ignore=sdk/python/tests/integration/offline_store/test_dqm_validation.py
+
 test-python-universal-singlestore-online:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.singlestore_repo_configuration \
@@ -549,43 +570,60 @@ build-ui:
 
 
 # Go SDK & embedded
-install-protoc-dependencies:
-	pip install "protobuf>=4.24.0,<5.0.0" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1"
+PB_REL = https://github.com/protocolbuffers/protobuf/releases
+PB_VERSION = 3.11.2
+PB_ARCH := $(shell uname -m)
+ifeq ($(PB_ARCH), arm64)
+	PB_ARCH=aarch_64
+endif
+PB_PROTO_FOLDERS=core registry serving types storage
 
-install-go-proto-dependencies:
+$(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip: $(TOOL_DIR)
+	cd $(TOOL_DIR) && \
+	curl -LO $(PB_REL)/download/v$(PB_VERSION)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip
+
+.PHONY: install-go-proto-dependencies
+install-go-proto-dependencies: $(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip
+	unzip -u $(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip -d $(TOOL_DIR)
 	go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0
 	go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0
 
+.PHONY: compile-protos-go
+compile-protos-go: install-go-proto-dependencies
+	$(foreach folder,$(PB_PROTO_FOLDERS), \
+		protoc --proto_path=$(ROOT_DIR)/protos \
+			--go_out=$(ROOT_DIR)/go/protos \
+			--go_opt=module=github.com/feast-dev/feast/go/protos \
+			--go-grpc_out=$(ROOT_DIR)/go/protos \
+			--go-grpc_opt=module=github.com/feast-dev/feast/go/protos $(ROOT_DIR)/protos/feast/$(folder)/*.proto; ) true
+
 #install-go-ci-dependencies:
 	# go install golang.org/x/tools/cmd/goimports
 	# python -m pip install "pybindgen==0.22.1" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1"
 
-build-go: 
-	compile-protos-go 
+.PHONY: build-go
+build-go: compile-protos-go 
 	go build -o feast ./go/main.go
 
+.PHONY: install-feast-ci-locally
 install-feast-ci-locally:
-	pip install -e ".[ci]"
+	uv pip install -e ".[ci]"
 
-test-go: 
-	compile-protos-go 
-	compile-protos-python 
-	install-feast-ci-locally
+.PHONY: test-go
+test-go: compile-protos-go install-feast-ci-locally compile-protos-python  
 	CGO_ENABLED=1 go test -coverprofile=coverage.out ./... && go tool cover -html=coverage.out -o coverage.html
 
+.PHONY: format-go
 format-go:
 	gofmt -s -w go/
 
-lint-go: 
-	compile-protos-go
+.PHONY: lint-go
+lint-go: compile-protos-go
 	go vet ./go/internal/feast
 
+.PHONY: build-go-docker-dev
 build-go-docker-dev:
 	docker buildx build --build-arg VERSION=dev \
 		-t feastdev/feature-server-go:dev \
 		-f go/infra/docker/feature-server/Dockerfile --load .
 
-compile-protos-go: 
-	install-go-proto-dependencies 
-	install-protoc-dependencies
-	python setup.py build_go_protos