diff --git a/development/stream_interface/workflows_demo.py b/development/stream_interface/workflows_demo.py index 5bfa7501c..b7fc41a34 100644 --- a/development/stream_interface/workflows_demo.py +++ b/development/stream_interface/workflows_demo.py @@ -11,7 +11,7 @@ from inference.core.utils.drawing import create_tiles STOP = False -ANNOTATOR = sv.BoundingBoxAnnotator() +ANNOTATOR = sv.BoxAnnotator() fps_monitor = sv.FPSMonitor() diff --git a/development/workflows_examples/video_analysis/aggregating_objects_passing_data.py b/development/workflows_examples/video_analysis/aggregating_objects_passing_data.py index db7702bda..73d1a5138 100644 --- a/development/workflows_examples/video_analysis/aggregating_objects_passing_data.py +++ b/development/workflows_examples/video_analysis/aggregating_objects_passing_data.py @@ -122,7 +122,7 @@ } STOP = False -ANNOTATOR = sv.BoundingBoxAnnotator() +ANNOTATOR = sv.BoxAnnotator() fps_monitor = sv.FPSMonitor() diff --git a/development/workflows_examples/video_analysis/using_webhook_sink.py b/development/workflows_examples/video_analysis/using_webhook_sink.py index 6201c7124..836de0670 100644 --- a/development/workflows_examples/video_analysis/using_webhook_sink.py +++ b/development/workflows_examples/video_analysis/using_webhook_sink.py @@ -208,7 +208,7 @@ } STOP = False -ANNOTATOR = sv.BoundingBoxAnnotator() +ANNOTATOR = sv.BoxAnnotator() fps_monitor = sv.FPSMonitor() diff --git a/docker/dockerfiles/Dockerfile.onnx.cpu b/docker/dockerfiles/Dockerfile.onnx.cpu index 5cb200532..707b4c40a 100644 --- a/docker/dockerfiles/Dockerfile.onnx.cpu +++ b/docker/dockerfiles/Dockerfile.onnx.cpu @@ -39,7 +39,7 @@ RUN pip3 install --upgrade pip && pip3 install \ -r requirements.yolo_world.txt \ -r requirements.transformers.txt \ jupyterlab \ - wheel>=0.38.0 \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip @@ -51,7 +51,7 @@ COPY --from=base / / WORKDIR /build COPY . . RUN make create_wheels -RUN pip3 install dist/inference_cli*.whl dist/inference_core*.whl dist/inference_cpu*.whl dist/inference_sdk*.whl +RUN pip3 install dist/inference_cli*.whl dist/inference_core*.whl dist/inference_cpu*.whl dist/inference_sdk*.whl "setuptools<=75.5.0" RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then pip3 install -r requirements/requirements.vino.txt; rm -rf ~/.cache/pip; fi diff --git a/docker/dockerfiles/Dockerfile.onnx.cpu.dev b/docker/dockerfiles/Dockerfile.onnx.cpu.dev index 32c19370c..8645c9ab2 100644 --- a/docker/dockerfiles/Dockerfile.onnx.cpu.dev +++ b/docker/dockerfiles/Dockerfile.onnx.cpu.dev @@ -39,7 +39,7 @@ RUN pip3 install --upgrade pip && pip3 install \ -r requirements.yolo_world.txt \ -r requirements.transformers.txt \ jupyterlab \ - wheel>=0.38.0 \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip @@ -51,7 +51,7 @@ COPY --from=base / / WORKDIR /build COPY . . RUN make create_wheels -RUN pip3 install dist/inference_cli*.whl dist/inference_core*.whl dist/inference_cpu*.whl dist/inference_sdk*.whl +RUN pip3 install dist/inference_cli*.whl dist/inference_core*.whl dist/inference_cpu*.whl dist/inference_sdk*.whl "setuptools<=75.5.0" RUN pip3 install watchdog[watchmedo] RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then pip3 install -r requirements/requirements.vino.txt; rm -rf ~/.cache/pip; fi diff --git a/docker/dockerfiles/Dockerfile.onnx.cpu.parallel b/docker/dockerfiles/Dockerfile.onnx.cpu.parallel index 64f3d770b..4da451f95 100644 --- a/docker/dockerfiles/Dockerfile.onnx.cpu.parallel +++ b/docker/dockerfiles/Dockerfile.onnx.cpu.parallel @@ -42,7 +42,7 @@ RUN pip3 install --upgrade pip && pip3 install \ -r requirements.parallel.txt \ -r requirements.cli.txt \ -r requirements.sdk.http.txt \ - wheel>=0.38.0 \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip RUN apt-get update && apt-get install -y lsb-release curl gpg diff --git a/docker/dockerfiles/Dockerfile.onnx.cpu.slim b/docker/dockerfiles/Dockerfile.onnx.cpu.slim index 855186bb4..9e421a13e 100644 --- a/docker/dockerfiles/Dockerfile.onnx.cpu.slim +++ b/docker/dockerfiles/Dockerfile.onnx.cpu.slim @@ -32,7 +32,7 @@ RUN pip3 install --upgrade pip && pip3 install \ -r requirements.waf.txt \ -r requirements.cli.txt \ -r requirements.sdk.http.txt \ - wheel>=0.38.0 \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.cpu.stream_manager b/docker/dockerfiles/Dockerfile.onnx.cpu.stream_manager index d1003dedf..ead560587 100644 --- a/docker/dockerfiles/Dockerfile.onnx.cpu.stream_manager +++ b/docker/dockerfiles/Dockerfile.onnx.cpu.stream_manager @@ -23,6 +23,7 @@ RUN pip3 install --upgrade pip && pip3 install \ -r _requirements.txt \ -r requirements.cpu.txt \ -r requirements.http.txt \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.gpu b/docker/dockerfiles/Dockerfile.onnx.gpu index 6f4162f41..1d7b8c5dc 100644 --- a/docker/dockerfiles/Dockerfile.onnx.gpu +++ b/docker/dockerfiles/Dockerfile.onnx.gpu @@ -44,6 +44,7 @@ RUN python3 -m pip install \ -r requirements.yolo_world.txt \ -r requirements.transformers.txt \ jupyterlab \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip @@ -60,7 +61,7 @@ WORKDIR /build COPY . . RUN ln -s /usr/bin/python3 /usr/bin/python RUN /bin/make create_wheels_for_gpu_notebook -RUN pip3 install --no-cache-dir dist/inference_cli*.whl dist/inference_core*.whl dist/inference_gpu*.whl dist/inference_sdk*.whl +RUN pip3 install --no-cache-dir dist/inference_cli*.whl dist/inference_core*.whl dist/inference_gpu*.whl dist/inference_sdk*.whl "setuptools<=75.5.0" WORKDIR /notebooks COPY examples/notebooks . diff --git a/docker/dockerfiles/Dockerfile.onnx.gpu.dev b/docker/dockerfiles/Dockerfile.onnx.gpu.dev index aaf43e640..211cb8c92 100644 --- a/docker/dockerfiles/Dockerfile.onnx.gpu.dev +++ b/docker/dockerfiles/Dockerfile.onnx.gpu.dev @@ -49,6 +49,7 @@ RUN python3 -m pip install \ -r requirements.sdk.http.txt \ -r requirements.cli.txt \ jupyterlab \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.gpu.parallel b/docker/dockerfiles/Dockerfile.onnx.gpu.parallel index 71da06f63..738ba8cd8 100644 --- a/docker/dockerfiles/Dockerfile.onnx.gpu.parallel +++ b/docker/dockerfiles/Dockerfile.onnx.gpu.parallel @@ -31,6 +31,7 @@ RUN pip3 install --upgrade pip && pip3 install \ -r requirements.waf.txt \ -r requirements.gaze.txt \ -r requirements.parallel.txt \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip @@ -47,7 +48,7 @@ WORKDIR /build COPY . . RUN ln -s /usr/bin/python3 /usr/bin/python RUN /bin/make create_wheels_for_gpu_notebook -RUN pip3 install dist/inference_cli*.whl dist/inference_core*.whl dist/inference_gpu*.whl dist/inference_sdk*.whl +RUN pip3 install dist/inference_cli*.whl dist/inference_core*.whl dist/inference_gpu*.whl dist/inference_sdk*.whl "setuptools<=75.5.0" WORKDIR /notebooks COPY examples/notebooks . diff --git a/docker/dockerfiles/Dockerfile.onnx.gpu.slim b/docker/dockerfiles/Dockerfile.onnx.gpu.slim index c12c3ba14..1739789c8 100644 --- a/docker/dockerfiles/Dockerfile.onnx.gpu.slim +++ b/docker/dockerfiles/Dockerfile.onnx.gpu.slim @@ -27,7 +27,7 @@ RUN pip3 install --upgrade pip && pip3 install \ -r requirements.waf.txt \ -r requirements.cli.txt \ -r requirements.sdk.http.txt \ - wheel>=0.38.0 \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.gpu.stream_manager b/docker/dockerfiles/Dockerfile.onnx.gpu.stream_manager index 597ea91bc..810f729bf 100644 --- a/docker/dockerfiles/Dockerfile.onnx.gpu.stream_manager +++ b/docker/dockerfiles/Dockerfile.onnx.gpu.stream_manager @@ -21,6 +21,7 @@ RUN pip3 install --upgrade pip && pip3 install \ -r _requirements.txt \ -r requirements.http.txt \ -r requirements.gpu.txt \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.jetson.4.5.0 b/docker/dockerfiles/Dockerfile.onnx.jetson.4.5.0 index cd8940d25..4d9bd9d04 100644 --- a/docker/dockerfiles/Dockerfile.onnx.jetson.4.5.0 +++ b/docker/dockerfiles/Dockerfile.onnx.jetson.4.5.0 @@ -42,6 +42,7 @@ RUN python3.8 -m pip install --upgrade pip wheel Cython && python3.8 -m pip inst -r requirements.sdk.http.txt \ -r requirements.yolo_world.txt \ jupyterlab \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.jetson.4.6.1 b/docker/dockerfiles/Dockerfile.onnx.jetson.4.6.1 index e8a7cae05..8c320d33d 100644 --- a/docker/dockerfiles/Dockerfile.onnx.jetson.4.6.1 +++ b/docker/dockerfiles/Dockerfile.onnx.jetson.4.6.1 @@ -57,6 +57,7 @@ RUN python3.9 -m pip install --upgrade pip "h5py<=3.10.0" && python3.9 -m pip in -r requirements.sdk.http.txt \ -r requirements.yolo_world.txt \ jupyterlab \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1 b/docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1 index 308a89af7..f26dce4f2 100644 --- a/docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1 +++ b/docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1 @@ -52,6 +52,7 @@ RUN python3.9 -m pip install --upgrade pip && python3.9 -m pip install \ -r requirements.sdk.http.txt \ -r requirements.yolo_world.txt \ jupyterlab \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1.stream_manager b/docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1.stream_manager index 5a11b799f..a5ee7a545 100644 --- a/docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1.stream_manager +++ b/docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1.stream_manager @@ -6,6 +6,8 @@ ENV LANG en_US.UTF-8 RUN apt-get update -y && apt-get install -y \ lshw \ git \ + python3.9 \ + python3.9-dev \ python3-pip \ python3-matplotlib \ gfortran \ @@ -24,30 +26,31 @@ COPY requirements/requirements.clip.txt \ requirements/_requirements.txt \ ./ -RUN pip3 install --ignore-installed PyYAML && rm -rf ~/.cache/pip +RUN python3.9 -m pip install --ignore-installed PyYAML && rm -rf ~/.cache/pip # We needed to take statically compiled library for last known stable build and put it into hosting # That was due to faulty builds started 26.06.2024, probably due to release of new version # of pybind11, which gets automatically pulled while build of zxing_cpp library making # cmake to fail -RUN wget https://storage.googleapis.com/roboflow-tests-assets/zxing_cpp_library_compiled_for_inference_v0.12.1_python_3.8.tar.gz \ - && tar -xvzf zxing_cpp_library_compiled_for_inference_v0.12.1_python_3.8.tar.gz \ - && mv zxing_cpp-2.2.0.dist-info /usr/local/lib/python3.8/dist-packages/zxing_cpp-2.2.0.dist-info \ - && mv zxingcpp.cpython-38-aarch64-linux-gnu.so /usr/local/lib/python3.8/dist-packages/ \ - && rm zxing_cpp_library_compiled_for_inference_v0.12.1_python_3.8.tar.gz +RUN wget https://storage.googleapis.com/roboflow-tests-assets/zxing_cpp_library_compiled_for_inference_v0.12.1.tar.gz \ + && tar -xvzf zxing_cpp_library_compiled_for_inference_v0.12.1.tar.gz \ + && mv zxing_cpp-2.2.0.dist-info /usr/local/lib/python3.9/dist-packages/zxing_cpp-2.2.0.dist-info \ + && mv zxingcpp.cpython-39-aarch64-linux-gnu.so /usr/local/lib/python3.9/dist-packages/ \ + && rm zxing_cpp_library_compiled_for_inference_v0.12.1.tar.gz -RUN pip3 install --upgrade pip && pip3 install \ +RUN python3.9 -m pip install --upgrade pip && python3.9 -m pip install \ -r _requirements.txt \ -r requirements.clip.txt \ -r requirements.http.txt \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip -RUN pip3 uninstall --yes onnxruntime -RUN wget https://nvidia.box.com/shared/static/iizg3ggrtdkqawkmebbfixo7sce6j365.whl -O onnxruntime_gpu-1.16.0-cp38-cp38-linux_aarch64.whl -RUN pip3 install onnxruntime_gpu-1.16.0-cp38-cp38-linux_aarch64.whl "opencv-python-headless<4.3" \ +RUN python3.9 -m pip uninstall --yes onnxruntime +RUN wget https://nvidia.box.com/shared/static/67zek28z497hs9aev7xg2c1wngdeyv4h.whl -O onnxruntime_gpu-1.16.0-cp39-cp39-linux_aarch64.whl +RUN python3.9 -m pip install onnxruntime_gpu-1.16.0-cp39-cp39-linux_aarch64.whl "opencv-python-headless>4" \ && rm -rf ~/.cache/pip \ - && rm onnxruntime_gpu-1.16.0-cp38-cp38-linux_aarch64.whl + && rm onnxruntime_gpu-1.16.0-cp39-cp39-linux_aarch64.whl WORKDIR /app/ COPY inference inference @@ -65,4 +68,4 @@ ENV WORKFLOWS_STEP_EXECUTION_MODE=local ENV WORKFLOWS_MAX_CONCURRENT_STEPS=1 ENV SUPERVISON_DEPRECATION_WARNING=0 -ENTRYPOINT ["python3", "-m", "inference.enterprise.stream_management.manager.app"] +ENTRYPOINT ["python3.9", "-m", "inference.enterprise.stream_management.manager.app"] diff --git a/docker/dockerfiles/Dockerfile.onnx.jetson.6.0.0 b/docker/dockerfiles/Dockerfile.onnx.jetson.6.0.0 index 0cdbae7b7..2807037dc 100644 --- a/docker/dockerfiles/Dockerfile.onnx.jetson.6.0.0 +++ b/docker/dockerfiles/Dockerfile.onnx.jetson.6.0.0 @@ -34,7 +34,8 @@ RUN python3 -m pip install --upgrade pip && \ -r requirements/requirements.groundingdino.txt \ -r requirements/requirements.sdk.http.txt \ -r requirements/requirements.yolo_world.txt \ - -r requirements/requirements.jetson.txt + -r requirements/requirements.jetson.txt \ + "setuptools<=75.5.0" # Build the application WORKDIR /build @@ -45,7 +46,7 @@ RUN rm -f dist/* && \ python3 .release/pypi/inference.gpu.setup.py bdist_wheel && \ python3 .release/pypi/inference.sdk.setup.py bdist_wheel && \ python3 .release/pypi/inference.cli.setup.py bdist_wheel && \ - python3 -m pip install dist/inference_cli*.whl dist/inference_core*.whl dist/inference_gpu*.whl dist/inference_sdk*.whl + python3 -m pip install dist/inference_cli*.whl dist/inference_core*.whl dist/inference_gpu*.whl dist/inference_sdk*.whl "setuptools<=75.5.0" # Set up the application runtime WORKDIR /app diff --git a/docker/dockerfiles/Dockerfile.onnx.lambda b/docker/dockerfiles/Dockerfile.onnx.lambda index 8c8a33a3b..4bb7e2b17 100644 --- a/docker/dockerfiles/Dockerfile.onnx.lambda +++ b/docker/dockerfiles/Dockerfile.onnx.lambda @@ -47,6 +47,7 @@ RUN pip3 install \ -r requirements.sdk.http.txt \ -r requirements.yolo_world.txt \ mangum \ + "setuptools<=75.5.0" \ --upgrade \ --target "${LAMBDA_TASK_ROOT}" \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.lambda.slim b/docker/dockerfiles/Dockerfile.onnx.lambda.slim index 3409bd0f5..12c746b5e 100644 --- a/docker/dockerfiles/Dockerfile.onnx.lambda.slim +++ b/docker/dockerfiles/Dockerfile.onnx.lambda.slim @@ -35,6 +35,7 @@ RUN pip3 install \ -r requirements.hosted.txt \ -r requirements.sdk.http.txt \ mangum \ + "setuptools<=75.5.0" \ --upgrade \ --target "${LAMBDA_TASK_ROOT}" \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.trt b/docker/dockerfiles/Dockerfile.onnx.trt index adf04ae3a..c57117f73 100644 --- a/docker/dockerfiles/Dockerfile.onnx.trt +++ b/docker/dockerfiles/Dockerfile.onnx.trt @@ -33,6 +33,7 @@ RUN pip install --upgrade pip && pip install \ -r requirements.doctr.txt \ -r requirements.groundingdino.txt \ -r requirements.sdk.http.txt \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.onnx.udp.gpu b/docker/dockerfiles/Dockerfile.onnx.udp.gpu index f86f39f48..efd79f235 100644 --- a/docker/dockerfiles/Dockerfile.onnx.udp.gpu +++ b/docker/dockerfiles/Dockerfile.onnx.udp.gpu @@ -25,6 +25,7 @@ RUN pip3 install --upgrade pip && pip3 install \ -r requirements.clip.txt \ -r requirements.http.txt \ -r requirements.gpu.txt \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docker/dockerfiles/Dockerfile.stream_management_api b/docker/dockerfiles/Dockerfile.stream_management_api index 8ff63f0c1..7f9c8ef5c 100644 --- a/docker/dockerfiles/Dockerfile.stream_management_api +++ b/docker/dockerfiles/Dockerfile.stream_management_api @@ -23,6 +23,7 @@ RUN pip3 install --upgrade pip && pip3 install \ -r _requirements.txt \ -r requirements.cpu.txt \ -r requirements.http.txt \ + "setuptools<=75.5.0" \ --upgrade \ && rm -rf ~/.cache/pip diff --git a/docs/foundation/paligemma.md b/docs/foundation/paligemma.md index 7b312b102..eb274f721 100644 --- a/docs/foundation/paligemma.md +++ b/docs/foundation/paligemma.md @@ -122,7 +122,7 @@ print(response) detections = from_pali_gemma(response=response, resolution_wh=image.size, class_list=['person', 'car', 'backpack']) -bounding_box_annotator = sv.BoundingBoxAnnotator() +bounding_box_annotator = sv.BoxAnnotator() label_annotator = sv.LabelAnnotator() annotatrd_image = bounding_box_annotator.annotate(image, detections) diff --git a/docs/foundation/yolo_world.md b/docs/foundation/yolo_world.md index 9e5921411..54c90cec1 100644 --- a/docs/foundation/yolo_world.md +++ b/docs/foundation/yolo_world.md @@ -43,7 +43,7 @@ YOLO World is faster than many other zero-shot object detection models like YOLO detections = sv.Detections.from_inference(results) - bounding_box_annotator = sv.BoundingBoxAnnotator() + bounding_box_annotator = sv.BoxAnnotator() label_annotator = sv.LabelAnnotator() labels = [classes[class_id] for class_id in detections.class_id] @@ -98,7 +98,7 @@ YOLO World is faster than many other zero-shot object detection models like YOLO detections = sv.Detections.from_inference(results) - bounding_box_annotator = sv.BoundingBoxAnnotator() + bounding_box_annotator = sv.BoxAnnotator() label_annotator = sv.LabelAnnotator() labels = [classes[class_id] for class_id in detections.class_id] diff --git a/docs/inference_helpers/inference_cli.md b/docs/inference_helpers/inference_cli.md index 55be649aa..8def634c1 100644 --- a/docs/inference_helpers/inference_cli.md +++ b/docs/inference_helpers/inference_cli.md @@ -234,8 +234,8 @@ inference infer -i {path_to_your_video_file} -m {your_project}/{version} -o {pat #### Configuration of visualisation Option `-c` can be provided with a path to `*.yml` file configuring `supervision` visualisation. There are few pre-defined configs: -- `bounding_boxes` - with `BoundingBoxAnnotator` and `LabelAnnotator` annotators -- `bounding_boxes_tracing` - with `ByteTracker` and annotators (`BoundingBoxAnnotator`, `LabelAnnotator`) +- `bounding_boxes` - with `BoxAnnotator` and `LabelAnnotator` annotators +- `bounding_boxes_tracing` - with `ByteTracker` and annotators (`BoxAnnotator`, `LabelAnnotator`) - `masks` - with `MaskAnnotator` and `LabelAnnotator` annotators - `polygons` - with `PolygonAnnotator` and `LabelAnnotator` annotators @@ -255,9 +255,9 @@ annotators: trace_length: 60 thickness: 2 tracking: - track_thresh: 0.25 - track_buffer: 30 - match_thresh: 0.8 + track_activation_threshold: 0.25 + lost_track_buffer: 30 + minimum_matching_threshold: 0.8 frame_rate: 30 ``` `annotators` field is a list of dictionaries with two keys: `type` and `param`. `type` points to @@ -265,7 +265,8 @@ name of annotator class: ```python from supervision import * ANNOTATOR_TYPE2CLASS = { - "bounding_box": BoundingBoxAnnotator, + "bounding_box": BoxAnnotator, + "box": BoxAnnotator, "mask": MaskAnnotator, "polygon": PolygonAnnotator, "color": ColorAnnotator, diff --git a/docs/notebooks/inference_pipeline_rtsp.ipynb b/docs/notebooks/inference_pipeline_rtsp.ipynb index 0b0eb5b36..9812426f4 100644 --- a/docs/notebooks/inference_pipeline_rtsp.ipynb +++ b/docs/notebooks/inference_pipeline_rtsp.ipynb @@ -151,7 +151,6 @@ "zones = [\n", " sv.PolygonZone(\n", " polygon=polygon,\n", - " frame_resolution_wh=[1440,2560],\n", " )\n", " for polygon\n", " in polygons\n", diff --git a/docs/quickstart/explore_models.md b/docs/quickstart/explore_models.md index b1c549eac..a331ca840 100644 --- a/docs/quickstart/explore_models.md +++ b/docs/quickstart/explore_models.md @@ -35,7 +35,7 @@ results = model.infer(image)[0] detections = sv.Detections.from_inference(results) # create supervision annotators -bounding_box_annotator = sv.BoundingBoxAnnotator() +bounding_box_annotator = sv.BoxAnnotator() label_annotator = sv.LabelAnnotator() # annotate the image with our inference results diff --git a/docs/quickstart/load_from_universe.md b/docs/quickstart/load_from_universe.md index 28c8c981f..483cf3694 100644 --- a/docs/quickstart/load_from_universe.md +++ b/docs/quickstart/load_from_universe.md @@ -50,7 +50,7 @@ results = model.infer(image) detections = sv.Detections.from_inference(results[0].dict(by_alias=True, exclude_none=True)) # create supervision annotators -bounding_box_annotator = sv.BoundingBoxAnnotator() +bounding_box_annotator = sv.BoxAnnotator() label_annotator = sv.LabelAnnotator() # annotate the image with our inference results diff --git a/docs/quickstart/run_model_on_rtsp_webcam.md b/docs/quickstart/run_model_on_rtsp_webcam.md index 6bc4d9cce..06d2244b7 100644 --- a/docs/quickstart/run_model_on_rtsp_webcam.md +++ b/docs/quickstart/run_model_on_rtsp_webcam.md @@ -126,7 +126,7 @@ import supervision as sv # create a bounding box annotator and label annotator to use in our custom sink label_annotator = sv.LabelAnnotator() -box_annotator = sv.BoundingBoxAnnotator() +box_annotator = sv.BoxAnnotator() def my_custom_sink(predictions: dict, video_frame: VideoFrame): # get the text labels for each prediction diff --git a/docs/using_inference/http_api.md b/docs/using_inference/http_api.md index 4cda003e4..c60cdd666 100644 --- a/docs/using_inference/http_api.md +++ b/docs/using_inference/http_api.md @@ -46,6 +46,10 @@ results = client.infer(image_url, model_id=model_id) ### Visualize Results ```python +import os + +import cv2 +import supervision as sv from inference_sdk import InferenceHTTPClient, InferenceConfiguration model_id = "soccer-players-5fuqs/1" @@ -66,7 +70,7 @@ result = client.infer(image, model_id=model_id) detections = sv.Detections.from_inference(result) #Create Supervision annotators -bounding_box_annotator = sv.BoundingBoxAnnotator() +bounding_box_annotator = sv.BoxAnnotator() label_annotator = sv.LabelAnnotator() #Extract labels array from inference results diff --git a/docs/using_inference/native_python_api.md b/docs/using_inference/native_python_api.md index 96fe087f9..b00f519a6 100644 --- a/docs/using_inference/native_python_api.md +++ b/docs/using_inference/native_python_api.md @@ -54,7 +54,7 @@ results = model.infer(image)[0] detections = sv.Detections.from_inference(results) # Create Supervision annotators -bounding_box_annotator = sv.BoundingBoxAnnotator() +bounding_box_annotator = sv.BoxAnnotator() label_annotator = sv.LabelAnnotator() # Extract labels array from inference results diff --git a/docs/workflows/blocks.md b/docs/workflows/blocks.md index 56e2748da..e69e3ee5e 100644 --- a/docs/workflows/blocks.md +++ b/docs/workflows/blocks.md @@ -95,6 +95,7 @@ hide:

+

diff --git a/docs/workflows/blocks_bundling.md b/docs/workflows/blocks_bundling.md index fd565b792..d03e2c53f 100644 --- a/docs/workflows/blocks_bundling.md +++ b/docs/workflows/blocks_bundling.md @@ -108,6 +108,65 @@ REGISTERED_INITIALIZERS = { } ``` +## Serializers and deserializers for *Kinds* + +Support for custom serializers and deserializers was introduced in Execution Engine `v1.3.0`. +From that version onward it is possible to point custom functions that +Execution Engine should use to serialize and deserialize any *[kind](/workflows/kinds/)*. + +Deserializers will determine how to decode inputs send through the wire +into internal data representation used by blocks. Serializers, on the other hand, +are useful when Workflow results are to be send through the wire. + +Below you may find example on how to add serializer and deserializer +for arbitrary kind. The code should be placed in main `__init__.py` of +your plugin: + +```python +from typing import Any + +def serialize_kind(value: Any) -> Any: + # place here the code that will be used to + # transform internal Workflows data representation into + # the external one (that can be sent through the wire in JSON, using + # default JSON encoder for Python). + pass + + +def deserialize_kind(parameter_name: str, value: Any) -> Any: + # place here the code that will be used to decode + # data sent through the wire into the Execution Engine + # and transform it into proper internal Workflows data representation + # which is understood by the blocks. + pass + + +KINDS_SERIALIZERS = { + "name_of_the_kind": serialize_kind, +} +KINDS_DESERIALIZERS = { + "name_of_the_kind": deserialize_kind, +} +``` + +### Tips And Tricks + +* Each serializer must be a function taking the value to serialize +and returning serialized value (accepted by default Python JSON encoder) + +* Each deserializer must be a function accepting two parameters - name of +Workflow input to be deserialized and the value to be deserialized - the goal +of the function is to align input data with expected internal representation + +* *Kinds* from `roboflow_core` plugin already have reasonable serializers and +deserializers + +* If you do not like the way how data is serialized in `roboflow_core` plugin, +feel free to alter the serialization methods for *kinds*, simply registering +the function in your plugin and loading it to the Execution Engine - the +serializer/deserializer defined as the last one will be in use. + + ## Enabling plugin in your Workflows ecosystem To load a plugin you must: diff --git a/docs/workflows/create_workflow_block.md b/docs/workflows/create_workflow_block.md index eaf01d1c2..bb01f7139 100644 --- a/docs/workflows/create_workflow_block.md +++ b/docs/workflows/create_workflow_block.md @@ -304,54 +304,57 @@ parsing specific steps in a Workflow definition * `name` - this property will be used to give the step a unique name and let other steps selects it via selectors -### Adding batch-oriented inputs +### Adding inputs -We want our step to take two batch-oriented inputs with images to be compared - so effectively -we will be creating SIMD block. +We want our step to take two inputs with images to be compared. -??? example "Adding batch-oriented inputs" +??? example "Adding inputs" Let's see how to add definitions of those inputs to manifest: - ```{ .py linenums="1" hl_lines="2 6 7 8 9 17 18 19 20 21 22"} + ```{ .py linenums="1" hl_lines="2 6-9 20-25"} from typing import Literal, Union from pydantic import Field from inference.core.workflows.prototypes.block import ( WorkflowBlockManifest, ) from inference.core.workflows.execution_engine.entities.types import ( - StepOutputImageSelector, - WorkflowImageSelector, + Selector, + IMAGE_KIND, ) + class ImagesSimilarityManifest(WorkflowBlockManifest): type: Literal["my_plugin/images_similarity@v1"] name: str # all properties apart from `type` and `name` are treated as either - # definitions of batch-oriented data to be processed by block or its - # parameters that influence execution of steps created based on block - image_1: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + # hardcoded parameters or data selectors. Data selectors are strings + # that start from `$steps.` or `$inputs.` marking references for data + # available in runtime - in this case we usually specify kinds of data + # to let compiler know what we expect the data to look like. + image_1: Selector(kind=[IMAGE_KIND]) = Field( description="First image to calculate similarity", ) - image_2: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_2: Selector(kind=[IMAGE_KIND]) = Field( description="Second image to calculate similarity", ) ``` * in the lines `2-9`, we've added a couple of imports to ensure that we have everything needed - * line `17` defines `image_1` parameter - as manifest is prototype for Workflow Definition, + * line `20` defines `image_1` parameter - as manifest is prototype for Workflow Definition, the only way to tell about image to be used by step is to provide selector - we have - two specialised types in core library that can be used - `WorkflowImageSelector` and `StepOutputImageSelector`. - If you look deeper into codebase, you will discover those are type aliases - telling `pydantic` + a specialised type in core library that can be used - `Selector`. + If you look deeper into codebase, you will discover this is type alias constructor function - telling `pydantic` to expect string matching `$inputs.{name}` and `$steps.{name}.*` patterns respectively, additionally providing extra schema field metadata that tells Workflows ecosystem components that the `kind` of data behind selector is - [image](/workflows/kinds/image/). + [image](/workflows/kinds/image/). **important note:** we denote *kind* as list - the list of specific kinds + is interpreted as *union of kinds* by Execution Engine. - * denoting `pydantic` `Field(...)` attribute in the last parts of line `17` is optional, yet appreciated, + * denoting `pydantic` `Field(...)` attribute in the last parts of line `20` is optional, yet appreciated, especially for blocks intended to cooperate with Workflows UI - * starting in line `20`, you can find definition of `image_2` parameter which is very similar to `image_1`. + * starting in line `23`, you can find definition of `image_2` parameter which is very similar to `image_1`. Such definition of manifest can handle the following step declaration in Workflow definition: @@ -367,75 +370,65 @@ Such definition of manifest can handle the following step declaration in Workflo This definition will make the Compiler and Execution Engine: -* select as a step prototype the block which declared manifest with type discriminator being -`my_plugin/images_similarity@v1` +* initialize the step from Workflow block declaring type `my_plugin/images_similarity@v1` * supply two parameters for the steps run method: - * `input_1` of type `WorkflowImageData` which will be filled with image submitted as Workflow execution input + * `input_1` of type `WorkflowImageData` which will be filled with image submitted as Workflow execution input + named `my_image`. * `imput_2` of type `WorkflowImageData` which will be generated at runtime, by another step called `image_transformation` -### Adding parameter to the manifest +### Adding parameters to the manifest -Let's now add the parameter that will influence step execution. The parameter is not assumed to be -batch-oriented and will affect all batch elements passed to the step. +Let's now add the parameter that will influence step execution. ??? example "Adding parameter to the manifest" - ```{ .py linenums="1" hl_lines="9 10 11 26 27 28 29 30 31 32"} + ```{ .py linenums="1" hl_lines="9 27-33"} from typing import Literal, Union from pydantic import Field from inference.core.workflows.prototypes.block import ( WorkflowBlockManifest, ) from inference.core.workflows.execution_engine.entities.types import ( - StepOutputImageSelector, - WorkflowImageSelector, - FloatZeroToOne, - WorkflowParameterSelector, + Selector, + IMAGE_KIND, FLOAT_ZERO_TO_ONE_KIND, ) + class ImagesSimilarityManifest(WorkflowBlockManifest): type: Literal["my_plugin/images_similarity@v1"] name: str # all properties apart from `type` and `name` are treated as either - # definitions of batch-oriented data to be processed by block or its - # parameters that influence execution of steps created based on block - image_1: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + # hardcoded parameters or data selectors. Data selectors are strings + # that start from `$steps.` or `$inputs.` marking references for data + # available in runtime - in this case we usually specify kinds of data + # to let compiler know what we expect the data to look like. + image_1: Selector(kind=[IMAGE_KIND]) = Field( description="First image to calculate similarity", ) - image_2: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_2: Selector(kind=[IMAGE_KIND]) = Field( description="Second image to calculate similarity", ) similarity_threshold: Union[ - FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + float, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Threshold to assume that images are similar", ) ``` - - * line `9` imports `FloatZeroToOne` which is type alias providing validation - for float values in range 0.0-1.0 - this is based on native `pydantic` mechanism and - everyone could create this type annotation locally in module hosting block - - * line `10` imports function `WorkflowParameterSelector(...)` capable to dynamically create - `pydantic` type annotation for selector to workflow input parameter (matching format `$inputs.param_name`), - declaring union of kinds compatible with the field - * line `11` imports [`float_zero_to_one`](/workflows/kinds/float_zero_to_one) `kind` definition which will be used later + * line `9` imports [`float_zero_to_one`](/workflows/kinds/float_zero_to_one) `kind` + definition which will be used to define the parameter. - * in line `26` we start defining parameter called `similarity_threshold`. Manifest will accept - either float values (in range `[0.0-1.0]`) or selector to workflow input of `kind` - [`float_zero_to_one`](/workflows/kinds/float_zero_to_one). Please point out on how - function creating type annotation (`WorkflowParameterSelector(...)`) is used - - in particular, expected `kind` of data is passed as list of `kinds` - representing union - of expected data `kinds`. + * in line `27` we start defining parameter called `similarity_threshold`. Manifest will accept + either float values or selector to workflow input of `kind` + [`float_zero_to_one`](/workflows/kinds/float_zero_to_one), imported in line `9`. Such definition of manifest can handle the following step declaration in Workflow definition: @@ -457,52 +450,14 @@ or alternatively: "name": "my_step", "image_1": "$inputs.my_image", "image_2": "$steps.image_transformation.image", - "similarity_threshold": "0.5" + "similarity_threshold": 0.5 } ``` -??? hint "LEARN MORE: Selecting step outputs" - - Our siplified example showcased declaration of properties that accept selectors to - images produced by other steps via `StepOutputImageSelector`. - - You can use function `StepOutputSelector(...)` creating field annotations dynamically - to express the that block accepts batch-oriented outputs from other steps of specified - kinds - - ```{ .py linenums="1" hl_lines="9 10 25"} - from typing import Literal, Union - from pydantic import Field - from inference.core.workflows.prototypes.block import ( - WorkflowBlockManifest, - ) - from inference.core.workflows.execution_engine.entities.types import ( - StepOutputImageSelector, - WorkflowImageSelector, - StepOutputSelector, - NUMPY_ARRAY_KIND, - ) - - class ImagesSimilarityManifest(WorkflowBlockManifest): - type: Literal["my_plugin/images_similarity@v1"] - name: str - # all properties apart from `type` and `name` are treated as either - # definitions of batch-oriented data to be processed by block or its - # parameters that influence execution of steps created based on block - image_1: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( - description="First image to calculate similarity", - ) - image_2: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( - description="Second image to calculate similarity", - ) - example: StepOutputSelector(kind=[NUMPY_ARRAY_KIND]) - ``` - ### Declaring block outputs -Our manifest is ready regarding properties that can be declared in Workflow definitions, -but we still need to provide additional information for the Execution Engine to successfully -run the block. +We have successfully defined inputs for our block, but we are still missing couple of elements required to +successfully run blocks. Let's define block outputs. ??? example "Declaring block outputs" @@ -510,34 +465,33 @@ run the block. to increase block stability, we advise to provide information about execution engine compatibility. - ```{ .py linenums="1" hl_lines="1 5 13 33-40 42-44"} - from typing import Literal, Union, List, Optional + ```{ .py linenums="1" hl_lines="5 11 32-39 41-43"} + from typing import Literal, Union from pydantic import Field from inference.core.workflows.prototypes.block import ( WorkflowBlockManifest, OutputDefinition, ) from inference.core.workflows.execution_engine.entities.types import ( - StepOutputImageSelector, - WorkflowImageSelector, - FloatZeroToOne, - WorkflowParameterSelector, + Selector, + IMAGE_KIND, FLOAT_ZERO_TO_ONE_KIND, BOOLEAN_KIND, ) + class ImagesSimilarityManifest(WorkflowBlockManifest): type: Literal["my_plugin/images_similarity@v1"] name: str - image_1: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_1: Selector(kind=[IMAGE_KIND]) = Field( description="First image to calculate similarity", ) - image_2: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_2: Selector(kind=[IMAGE_KIND]) = Field( description="Second image to calculate similarity", ) similarity_threshold: Union[ - FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + float, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Threshold to assume that images are similar", @@ -554,21 +508,19 @@ run the block. @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" ``` - - * line `1` contains additional imports from `typing` - + * line `5` imports class that is used to describe step outputs - * line `13` imports [`boolean`](/workflows/kinds/boolean) `kind` to be used + * line `11` imports [`boolean`](/workflows/kinds/boolean) `kind` to be used in outputs definitions - * lines `33-40` declare class method to specify outputs from the block - + * lines `32-39` declare class method to specify outputs from the block - each entry in list declare one return property for each batch element and its `kind`. Our block will return boolean flag `images_match` for each pair of images. - * lines `42-44` declare compatibility of the block with Execution Engine - + * lines `41-43` declare compatibility of the block with Execution Engine - see [versioning page](/workflows/versioning/) for more details As a result of those changes: @@ -591,7 +543,7 @@ in their inputs * additionally, block manifest should implement instance method `get_actual_outputs(...)` that provides list of actual outputs that can be generated based on filled manifest data - ```{ .py linenums="1" hl_lines="14 35-42 44-49"} + ```{ .py linenums="1" hl_lines="13 35-42 44-49"} from typing import Literal, Union, List, Optional from pydantic import Field from inference.core.workflows.prototypes.block import ( @@ -599,27 +551,27 @@ in their inputs OutputDefinition, ) from inference.core.workflows.execution_engine.entities.types import ( - StepOutputImageSelector, - WorkflowImageSelector, + Selector, + IMAGE_KIND, FloatZeroToOne, - WorkflowParameterSelector, FLOAT_ZERO_TO_ONE_KIND, BOOLEAN_KIND, WILDCARD_KIND, ) + class ImagesSimilarityManifest(WorkflowBlockManifest): type: Literal["my_plugin/images_similarity@v1"] name: str - image_1: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_1: Selector(kind=[IMAGE_KIND]) = Field( description="First image to calculate similarity", ) - image_2: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_2: Selector(kind=[IMAGE_KIND]) = Field( description="Second image to calculate similarity", ) similarity_threshold: Union[ - FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + float, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Threshold to assume that images are similar", @@ -657,7 +609,7 @@ block. ??? example "Block scaffolding" - ```{ .py linenums="1" hl_lines="1 5 6 8-11 56-68"} + ```{ .py linenums="1" hl_lines="1 5 6 8-11 53-55 57-63"} from typing import Literal, Union, List, Optional, Type from pydantic import Field from inference.core.workflows.prototypes.block import ( @@ -670,10 +622,9 @@ block. WorkflowImageData, ) from inference.core.workflows.execution_engine.entities.types import ( - StepOutputImageSelector, - WorkflowImageSelector, + Selector, + IMAGE_KIND, FloatZeroToOne, - WorkflowParameterSelector, FLOAT_ZERO_TO_ONE_KIND, BOOLEAN_KIND, ) @@ -681,15 +632,15 @@ block. class ImagesSimilarityManifest(WorkflowBlockManifest): type: Literal["my_plugin/images_similarity@v1"] name: str - image_1: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_1: Selector(kind=[IMAGE_KIND]) = Field( description="First image to calculate similarity", ) - image_2: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_2: Selector(kind=[IMAGE_KIND]) = Field( description="Second image to calculate similarity", ) similarity_threshold: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Threshold to assume that images are similar", @@ -706,7 +657,7 @@ block. @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ImagesSimilarityBlock(WorkflowBlock): @@ -724,15 +675,18 @@ block. pass ``` - * lines `1`, `5-6` and `8-9` added changes into import surtucture to + * lines `1`, `5-6` and `8-11` added changes into import surtucture to provide additional symbols required to properly define block class and all of its methods signatures - * line `59` defines class method `get_manifest(...)` to simply return + * lines `53-55` defines class method `get_manifest(...)` to simply return the manifest class we cretaed earlier - * lines `62-68` define `run(...)` function, which Execution Engine - will invoke with data to get desired results + * lines `57-63` define `run(...)` function, which Execution Engine + will invoke with data to get desired results. Please note that + manifest fields defining inputs of [image](/workflows/kinds/image/) kind + are marked as `WorkflowImageData` - which is compliant with intenal data + representation of `image` kind described in [kind documentation](/workflows/kinds/image/). ### Providing implementation for block logic @@ -747,7 +701,7 @@ it can produce meaningful results. ??? example "Implementation of `run(...)` method" - ```{ .py linenums="1" hl_lines="3 56-58 70-81"} + ```{ .py linenums="1" hl_lines="3 55-57 69-80"} from typing import Literal, Union, List, Optional, Type from pydantic import Field import cv2 @@ -762,10 +716,9 @@ it can produce meaningful results. WorkflowImageData, ) from inference.core.workflows.execution_engine.entities.types import ( - StepOutputImageSelector, - WorkflowImageSelector, + Selector, + IMAGE_KIND, FloatZeroToOne, - WorkflowParameterSelector, FLOAT_ZERO_TO_ONE_KIND, BOOLEAN_KIND, ) @@ -773,15 +726,15 @@ it can produce meaningful results. class ImagesSimilarityManifest(WorkflowBlockManifest): type: Literal["my_plugin/images_similarity@v1"] name: str - image_1: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_1: Selector(kind=[IMAGE_KIND]) = Field( description="First image to calculate similarity", ) - image_2: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_2: Selector(kind=[IMAGE_KIND]) = Field( description="Second image to calculate similarity", ) similarity_threshold: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Threshold to assume that images are similar", @@ -798,7 +751,7 @@ it can produce meaningful results. @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ImagesSimilarityBlock(WorkflowBlock): @@ -833,49 +786,30 @@ it can produce meaningful results. * in line `3` we import OpenCV - * lines `56-58` defines block constructor, thanks to this - state of block + * lines `55-57` defines block constructor, thanks to this - state of block is initialised once and live through consecutive invocation of `run(...)` method - for instance when Execution Engine runs on consecutive frames of video - * lines `70-81` provide implementation of block functionality - the details are trully not + * lines `69-80` provide implementation of block functionality - the details are trully not important regarding Workflows ecosystem, but there are few details you should focus: - * lines `70` and `71` make use of `WorkflowImageData` abstraction, showcasing how + * lines `69` and `70` make use of `WorkflowImageData` abstraction, showcasing how `numpy_image` property can be used to get `np.ndarray` from internal representation of images in Workflows. We advise to expole remaining properties of `WorkflowImageData` to discover more. - * result of workflow block execution, declared in lines `79-81` is in our case just a dictionary - **with the keys being the names of outputs declared in manifest**, in line `44`. Be sure to provide all + * result of workflow block execution, declared in lines `78-80` is in our case just a dictionary + **with the keys being the names of outputs declared in manifest**, in line `43`. Be sure to provide all declared outputs - otherwise Execution Engine will raise error. - -You may ask yourself how it is possible that implemented block accepts batch-oriented workflow input, but do not -operate on batches directly. This is due to the fact that the default block behaviour is to run one-by-one against -all elements of input batches. We will show how to change that in [advanced topics](#advanced-topics) section. - -!!! note - - One important note: blocks, like all other classes, have constructors that may initialize a state. This state can - persist across multiple Workflow runs when using the same instance of the Execution Engine. If the state management - needs to be aware of which batch element it processes (e.g., in object tracking scenarios), the block creator - should use dedicated batch-oriented inputs. These inputs, provide relevant metadatadata — like the - `WorkflowVideoMetadata` input, which is crucial for tracking use cases and can be used along with `WorkflowImage` - input in a block implementing tracker. - - The ecosystem is evolving, and new input types will be introduced over time. If a specific input type needed for - a use case is not available, an alternative is to design the block to process entire input batches. This way, - you can rely on the Batch container's indices property, which provides an index for each batch element, allowing - you to maintain the correct order of processing. ## Exposing block in `plugin` -Now, your block is ready to be used, but if you declared step using it in your Workflow definition you -would see an error. This is because no plugin exports the block you just created. Details of blocks bundling -will be covered in [separate page](/workflows/blocks_bundling/), but the remaining thing to do is to -add block class into list returned from your plugins' `load_blocks(...)` function: +Now, your block is ready to be used, but Execution Engine is not aware of its existence. This is because no registered +plugin exports the block you just created. Details of blocks bundling are be covered in [separate page](/workflows/blocks_bundling/), +but the remaining thing to do is to add block class into list returned from your plugins' `load_blocks(...)` function: ```python -# __init__.py of your plugin +# __init__.py of your plugin (or roboflow_core plugin if you contribute directly to `inference`) from my_plugin.images_similarity.v1 import ImagesSimilarityBlock # this is example import! requires adjustment @@ -895,7 +829,7 @@ on how to use it for your block. ??? example "Implementation of blocks accepting batches" - ```{ .py linenums="1" hl_lines="13 41-43 71-72 75-78 86-87"} + ```{ .py linenums="1" hl_lines="13 40-42 70-71 74-77 85-86"} from typing import Literal, Union, List, Optional, Type from pydantic import Field import cv2 @@ -911,10 +845,9 @@ on how to use it for your block. Batch, ) from inference.core.workflows.execution_engine.entities.types import ( - StepOutputImageSelector, - WorkflowImageSelector, + Selector, + IMAGE_KIND, FloatZeroToOne, - WorkflowParameterSelector, FLOAT_ZERO_TO_ONE_KIND, BOOLEAN_KIND, ) @@ -922,23 +855,23 @@ on how to use it for your block. class ImagesSimilarityManifest(WorkflowBlockManifest): type: Literal["my_plugin/images_similarity@v1"] name: str - image_1: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_1: Selector(kind=[IMAGE_KIND]) = Field( description="First image to calculate similarity", ) - image_2: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image_2: Selector(kind=[IMAGE_KIND]) = Field( description="Second image to calculate similarity", ) similarity_threshold: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Threshold to assume that images are similar", ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> bool: + return ["image_1", "image_2"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -951,7 +884,7 @@ on how to use it for your block. @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ImagesSimilarityBlock(WorkflowBlock): @@ -988,19 +921,125 @@ on how to use it for your block. * line `13` imports `Batch` from core of workflows library - this class represent container which is veri similar to list (but read-only) to keep batch elements - * lines `41-43` define class method that changes default behaviour of the block and make it capable - to process batches + * lines `40-42` define class method that changes default behaviour of the block and make it capable + to process batches - we are marking each parameter that the `run(...)` method **recognizes as batch-oriented**. * changes introduced above made the signature of `run(...)` method to change, now `image_1` and `image_2` - are not instances of `WorkflowImageData`, but rather batches of elements of this type + are not instances of `WorkflowImageData`, but rather batches of elements of this type. **Important note:** + having multiple batch-oriented parameters we expect that those batches would have the elements related to + each other at corresponding positions - such that our block comparing `image_1[1]` into `image_2[1]` actually + performs logically meaningful operation. - * lines `75-78`, `86-87` present changes that needed to be introduced to run processing across all batch + * lines `74-77`, `85-86` present changes that needed to be introduced to run processing across all batch elements - showcasing how to iterate over batch elements if needed - * it is important to note how outputs are constructed in line `86` - each element of batch will be given + * it is important to note how outputs are constructed in line `85` - each element of batch will be given its entry in the list which is returned from `run(...)` method. Order must be aligned with order of batch elements. Each output dictionary must provide all keys declared in block outputs. + +??? Warning "Inputs that accept both batches and scalars" + + It is **relatively unlikely**, but may happen that your block would need to accept both batch-oriented data + and scalars within a single input parameter. Execution Engine recognises that using + `get_parameters_accepting_batches_and_scalars(...)` method of block manifest. Take a look at the + example provided below: + + + ```{ .py linenums="1" hl_lines="20-22 24-26 45-47 49 50-54 65-70"} + from typing import Literal, Union, List, Optional, Type, Any, Dict + from pydantic import Field + + from inference.core.workflows.prototypes.block import ( + WorkflowBlockManifest, + WorkflowBlock, + BlockResult, + ) + from inference.core.workflows.execution_engine.entities.base import ( + OutputDefinition, + Batch, + ) + from inference.core.workflows.execution_engine.entities.types import ( + Selector, + ) + + class ExampleManifest(WorkflowBlockManifest): + type: Literal["my_plugin/example@v1"] + name: str + param_1: Selector() + param_2: List[Selector()] + param_3: Dict[str, Selector()] + + @classmethod + def get_parameters_accepting_batches_and_scalars(cls) -> bool: + return ["param_1", "param_2", "param_3"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [OutputDefinition(name="dummy")] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + + class ExampleBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return ExampleManifest + + def run( + self, + param_1: Any, + param_2: List[Any], + param_3: Dict[str, Any], + ) -> BlockResult: + batch_size = None + if isinstance(param_1, Batch): + param_1_result = ... # do something with batch-oriented param + batch_size = len(param_1) + else: + param_1_result = ... # do something with scalar param + for element in param_2: + if isinstance(element, Batch): + ... + else: + ... + for key, value in param_3.items(): + if isinstance(element, value): + ... + else: + ... + if batch_size is None: + return {"dummy": "some_result"} + result = [] + for _ in range(batch_size): + result.append({"dummy": "some_result"}) + return result + ``` + + * lines `20-22` specify manifest parameters that are expected to accept mixed (both scalar and batch-oriented) + input data - point out that at this stage there is no difference in definition compared to previous examples. + + * lines `24-26` specify `get_parameters_accepting_batches_and_scalars(...)` method to tell the Execution + Engine that block `run(...)` method can handle both scalar and batch-oriented inputs for the specified + parameters. + + * lines `45-47` depict the parameters of mixed nature in `run(...)` method signature. + + * line `49` reveals that we must keep track of the expected output size **within the block logic**. That's + why it is quite tricky to implement blocks with mixed inputs. Normally, when block `run(...)` method + operates on scalars - in majority of cases (exceptions will be described below) - the metod constructs + single output dictionary. Similairly, when batch-oriented inputs are accepted - those inputs + define expected output size. In this case, however, we must manually detect batches and catch their sizes. + + * lines `50-54` showcase how we usually deal with mixed parameters - applying different logic when + batch-oriented data is detected + + * as mentioned earlier, output construction must also be adjusted to the nature of mixed inputs - which + is illustrated in lines `65-70` + ### Implementation of flow-control block Flow-control blocks differs quite substantially from other blocks that just process the data. Here we will show @@ -1014,11 +1053,11 @@ is defined as `$steps.{step_name}` - similar to step output selector, but withou * `FlowControl` object specify next steps (from selectors provided in step manifest) that for given batch element (SIMD flow-control) or whole workflow execution (non-SIMD flow-control) should pick up next -??? example "Implementation of flow-control - SIMD block" +??? example "Implementation of flow-control" Example provides and comments out implementation of random continue block - ```{ .py linenums="1" hl_lines="10 14 26 28-31 55-56"} + ```{ .py linenums="1" hl_lines="10 14 28-31 55-56"} from typing import List, Literal, Optional, Type, Union import random @@ -1029,8 +1068,8 @@ batch element (SIMD flow-control) or whole workflow execution (non-SIMD flow-con ) from inference.core.workflows.execution_engine.entities.types import ( StepSelector, - WorkflowImageSelector, - StepOutputImageSelector, + Selector, + IMAGE_KIND, ) from inference.core.workflows.execution_engine.v1.entities import FlowControl from inference.core.workflows.prototypes.block import ( @@ -1044,7 +1083,7 @@ batch element (SIMD flow-control) or whole workflow execution (non-SIMD flow-con class BlockManifest(WorkflowBlockManifest): type: Literal["my_plugin/random_continue@v1"] name: str - image: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField + image: Selector(kind=[IMAGE_KIND]) = ImageInputField probability: float next_steps: List[StepSelector] = Field( description="Reference to step which shall be executed if expression evaluates to true", @@ -1057,7 +1096,7 @@ batch element (SIMD flow-control) or whole workflow execution (non-SIMD flow-con @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.2.0,<2.0.0" class RandomContinueBlockV1(WorkflowBlock): @@ -1083,30 +1122,30 @@ batch element (SIMD flow-control) or whole workflow execution (non-SIMD flow-con * line `14` imports `FlowControl` class which is the only viable response from flow-control block - * line `26` specifies `image` which is batch-oriented input making the block SIMD - - which means that for each element of images batch, block will make random choice on - flow-control - if not that input block would operate in non-SIMD mode - * line `28` defines list of step selectors **which effectively turns the block into flow-control one** * lines `55` and `56` show how to construct output - `FlowControl` object accept context being `None`, `string` or `list of strings` - `None` represent flow termination for the batch element, strings are expected to be selectors for next steps, passed in input. -??? example "Implementation of flow-control non-SIMD block" +??? example "Implementation of flow-control - batch variant" Example provides and comments out implementation of random continue block - ```{ .py linenums="1" hl_lines="9 11 24-27 50-51"} + ```{ .py linenums="1" hl_lines="8 11 15 29-32 38-40 55 59 60 61-63"} from typing import List, Literal, Optional, Type, Union import random from pydantic import Field from inference.core.workflows.execution_engine.entities.base import ( OutputDefinition, + WorkflowImageData, + Batch, ) from inference.core.workflows.execution_engine.entities.types import ( StepSelector, + Selector, + IMAGE_KIND, ) from inference.core.workflows.execution_engine.v1.entities import FlowControl from inference.core.workflows.prototypes.block import ( @@ -1120,6 +1159,7 @@ batch element (SIMD flow-control) or whole workflow execution (non-SIMD flow-con class BlockManifest(WorkflowBlockManifest): type: Literal["my_plugin/random_continue@v1"] name: str + image: Selector(kind=[IMAGE_KIND]) = ImageInputField probability: float next_steps: List[StepSelector] = Field( description="Reference to step which shall be executed if expression evaluates to true", @@ -1129,10 +1169,14 @@ batch element (SIMD flow-control) or whole workflow execution (non-SIMD flow-con @classmethod def describe_outputs(cls) -> List[OutputDefinition]: return [] + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["image"] @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RandomContinueBlockV1(WorkflowBlock): @@ -1143,23 +1187,34 @@ batch element (SIMD flow-control) or whole workflow execution (non-SIMD flow-con def run( self, + image: Batch[WorkflowImageData], probability: float, next_steps: List[str], ) -> BlockResult: - if not next_steps or random.random() > probability: - return FlowControl() - return FlowControl(context=next_steps) + result = [] + for _ in image: + if not next_steps or random.random() > probability: + result.append(FlowControl()) + result.append(FlowControl(context=next_steps)) + return result ``` - * line `9` imports type annotation for step selector which will be used to + * line `11` imports type annotation for step selector which will be used to notify Execution Engine that the block controls the flow - * line `11` imports `FlowControl` class which is the only viable response from + * line `15` imports `FlowControl` class which is the only viable response from flow-control block - * lines `24-27` defines list of step selectors **which effectively turns the block into flow-control one** + * lines `29-32` defines list of step selectors **which effectively turns the block into flow-control one** + + * lines `38-40` contain definition of `get_parameters_accepting_batches(...)` method telling Execution + Engine that block `run(...)` method expects batch-oriented `image` parameter. + + * line `59` revels that we need to return flow-control guide for each and every element of `image` batch. - * lines `50` and `51` show how to construct output - `FlowControl` object accept context being `None`, `string` or + * to achieve that end, in line `60` we iterate over the contntent of batch. + + * lines `61-63` show how to construct output - `FlowControl` object accept context being `None`, `string` or `list of strings` - `None` represent flow termination for the batch element, strings are expected to be selectors for next steps, passed in input. @@ -1196,7 +1251,7 @@ def run(self, predictions: List[dict]) -> BlockResult: OutputDefinition, ) from inference.core.workflows.execution_engine.entities.types import ( - StepOutputSelector, + Selector, OBJECT_DETECTION_PREDICTION_KIND, ) from inference.core.workflows.prototypes.block import ( @@ -1210,7 +1265,7 @@ def run(self, predictions: List[dict]) -> BlockResult: class BlockManifest(WorkflowBlockManifest): type: Literal["my_plugin/fusion_of_predictions@v1"] name: str - predictions: List[StepOutputSelector(kind=[OBJECT_DETECTION_PREDICTION_KIND])] = Field( + predictions: List[Selector(kind=[OBJECT_DETECTION_PREDICTION_KIND])] = Field( description="Selectors to step outputs", examples=[["$steps.model_1.predictions", "$steps.model_2.predictions"]], ) @@ -1226,7 +1281,7 @@ def run(self, predictions: List[dict]) -> BlockResult: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class FusionBlockV1(WorkflowBlock): @@ -1272,7 +1327,7 @@ keys serve as names for those selectors. ??? example "Nested selectors - named selectors" - ```{ .py linenums="1" hl_lines="23-26 47"} + ```{ .py linenums="1" hl_lines="22-25 46"} from typing import List, Literal, Optional, Type, Any from pydantic import Field @@ -1281,8 +1336,7 @@ keys serve as names for those selectors. OutputDefinition, ) from inference.core.workflows.execution_engine.entities.types import ( - StepOutputSelector, - WorkflowParameterSelector, + Selector ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -1295,7 +1349,7 @@ keys serve as names for those selectors. class BlockManifest(WorkflowBlockManifest): type: Literal["my_plugin/named_selectors_example@v1"] name: str - data: Dict[str, StepOutputSelector(), WorkflowParameterSelector()] = Field( + data: Dict[str, Selector()] = Field( description="Selectors to step outputs", examples=[{"a": $steps.model_1.predictions", "b": "$Inputs.data"}], ) @@ -1308,7 +1362,7 @@ keys serve as names for those selectors. @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class BlockWithNamedSelectorsV1(WorkflowBlock): @@ -1325,10 +1379,10 @@ keys serve as names for those selectors. return {"my_output": ...} ``` - * lines `23-26` depict how to define manifest field capable of accepting - list of selectors + * lines `22-25` depict how to define manifest field capable of accepting + dictionary of selectors - providing mapping between selector name and value - * line `47` shows what to expect as input to block's `run(...)` method - + * line `46` shows what to expect as input to block's `run(...)` method - dict of objects which are reffered with selectors. If the block accepted batches, the input type of `data` field would be `Dict[str, Union[Batch[Any], Any]]`. In non-batch cases, non-batch-oriented data referenced by selector is automatically @@ -1387,7 +1441,7 @@ the method signatures. In this example, we perform dynamic crop of image based on predictions. - ```{ .py linenums="1" hl_lines="30-32 65 66-67"} + ```{ .py linenums="1" hl_lines="28-30 63 64-65"} from typing import Dict, List, Literal, Optional, Type, Union from uuid import uuid4 @@ -1400,9 +1454,7 @@ the method signatures. from inference.core.workflows.execution_engine.entities.types import ( IMAGE_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -1412,8 +1464,8 @@ the method signatures. class BlockManifest(WorkflowBlockManifest): type: Literal["my_block/dynamic_crop@v1"] - image: Union[WorkflowImageSelector, StepOutputImageSelector] - predictions: StepOutputSelector( + image: Selector(kind=[IMAGE_KIND]) + predictions: Selector( kind=[OBJECT_DETECTION_PREDICTION_KIND], ) @@ -1429,7 +1481,7 @@ the method signatures. @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DynamicCropBlockV1(WorkflowBlock): @@ -1457,16 +1509,16 @@ the method signatures. return crops ``` - * in lines `30-32` manifest class declares output dimensionality + * in lines `28-30` manifest class declares output dimensionality offset - value `1` should be understood as adding `1` to dimensionality level - * point out, that in line `65`, block eliminates empty images from further processing but + * point out, that in line `63`, block eliminates empty images from further processing but placing `None` instead of dictionatry with outputs. This would utilise the same Execution Engine behaviour that is used for conditional execution - datapoint will be eliminated from downstream processing (unless steps requesting empty inputs are present down the line). - * in lines `66-67` results for single input `image` and `predictions` are collected - + * in lines `64-65` results for single input `image` and `predictions` are collected - it is meant to be list of dictionares containing all registered outputs as keys. Execution engine will understand that the step returns batch of elements for each input element and create nested sturcures of indices to keep track of during execution of downstream steps. @@ -1476,7 +1528,7 @@ the method signatures. In this example, the block visualises crops predictions and creates tiles presenting all crops predictions in single output image. - ```{ .py linenums="1" hl_lines="31-33 50-51 61-62"} + ```{ .py linenums="1" hl_lines="29-31 48-49 59-60"} from typing import List, Literal, Type, Union import supervision as sv @@ -1489,9 +1541,7 @@ the method signatures. from inference.core.workflows.execution_engine.entities.types import ( IMAGE_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -1502,8 +1552,8 @@ the method signatures. class BlockManifest(WorkflowBlockManifest): type: Literal["my_plugin/tile_detections@v1"] - crops: Union[WorkflowImageSelector, StepOutputImageSelector] - crops_predictions: StepOutputSelector( + crops: Selector(kind=[IMAGE_KIND]) + crops_predictions: Selector( kind=[OBJECT_DETECTION_PREDICTION_KIND] ) @@ -1529,7 +1579,7 @@ the method signatures. crops: Batch[WorkflowImageData], crops_predictions: Batch[sv.Detections], ) -> BlockResult: - annotator = sv.BoundingBoxAnnotator() + annotator = sv.BoxAnnotator() visualisations = [] for image, prediction in zip(crops, crops_predictions): annotated_image = annotator.annotate( @@ -1541,10 +1591,10 @@ the method signatures. return {"visualisations": tile} ``` - * in lines `31-33` manifest class declares output dimensionality + * in lines `29-31` manifest class declares output dimensionality offset - value `-1` should be understood as decreasing dimensionality level by `1` - * in lines `50-51` you can see the impact of output dimensionality decrease + * in lines `48-49` you can see the impact of output dimensionality decrease on the method signature. Both inputs are artificially wrapped in `Batch[]` container. This is done by Execution Engine automatically on output dimensionality decrease when all inputs have the same dimensionality to enable access to all elements occupying @@ -1552,7 +1602,7 @@ the method signatures. from top-level batch will be grouped. For instance, if you had two input images that you cropped - crops from those two different images will be grouped separately. - * lines `61-62` illustrate how output is constructed - single value is returned and that value + * lines `59-60` illustrate how output is constructed - single value is returned and that value will be indexed by Execution Engine in output batch with reduced dimensionality === "different input dimensionalities" @@ -1561,7 +1611,7 @@ the method signatures. crops of original image - result is to provide single detections with all partial ones being merged. - ```{ .py linenums="1" hl_lines="32-37 39-41 63-64 70"} + ```{ .py linenums="1" hl_lines="31-36 38-40 62-63 69"} from copy import deepcopy from typing import Dict, List, Literal, Optional, Type, Union @@ -1575,9 +1625,8 @@ the method signatures. ) from inference.core.workflows.execution_engine.entities.types import ( OBJECT_DETECTION_PREDICTION_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, + Selector, + IMAGE_KIND, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -1588,8 +1637,8 @@ the method signatures. class BlockManifest(WorkflowBlockManifest): type: Literal["my_plugin/stitch@v1"] - image: Union[WorkflowImageSelector, StepOutputImageSelector] - image_predictions: StepOutputSelector( + image: Selector(kind=[IMAGE_KIND]) + image_predictions: Selector( kind=[OBJECT_DETECTION_PREDICTION_KIND], ) @@ -1635,11 +1684,11 @@ the method signatures. ``` - * in lines `32-37` manifest class declares input dimensionalities offset, indicating + * in lines `31-36` manifest class declares input dimensionalities offset, indicating `image` parameter being top-level and `image_predictions` being nested batch of predictions * whenever different input dimensionalities are declared, dimensionality reference property - must be pointed (see lines `39-41`) - this dimensionality level would be used to calculate + must be pointed (see lines `38-40`) - this dimensionality level would be used to calculate output dimensionality - in this particular case, we specify `image`. This choice has an implication in the expected format of result - in the chosen scenario we are supposed to return single dictionary with all registered outputs keys. If our choice is `image_predictions`, @@ -1647,11 +1696,11 @@ the method signatures. `get_dimensionality_reference_property(...)` which dimensionality level should be associated to the output. - * lines `63-64` present impact of dimensionality offsets specified in lines `32-37`. It is clearly + * lines `63-64` present impact of dimensionality offsets specified in lines `31-36`. It is clearly visible that `image_predictions` is a nested batch regarding `image`. Obviously, only nested predictions relevant for the specific `images` are grouped in batch and provided to the method in runtime. - * as mentioned earlier, line `70` construct output being single dictionary, as we register output + * as mentioned earlier, line `69` construct output being single dictionary, as we register output at dimensionality level of `image` (which was also shipped as single element) @@ -1661,7 +1710,7 @@ the method signatures. In this example, we perform dynamic crop of image based on predictions. - ```{ .py linenums="1" hl_lines="31-33 35-37 57-58 72 73-75"} + ```{ .py linenums="1" hl_lines="29-31 33-35 55-56 70 71-73"} from typing import Dict, List, Literal, Optional, Type, Union from uuid import uuid4 @@ -1675,9 +1724,7 @@ the method signatures. from inference.core.workflows.execution_engine.entities.types import ( IMAGE_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -1687,14 +1734,14 @@ the method signatures. class BlockManifest(WorkflowBlockManifest): type: Literal["my_block/dynamic_crop@v1"] - image: Union[WorkflowImageSelector, StepOutputImageSelector] - predictions: StepOutputSelector( + image: Selector(kind=[IMAGE_KIND]) + predictions: Selector( kind=[OBJECT_DETECTION_PREDICTION_KIND], ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> bool: + return ["image", "predictions"] @classmethod def get_output_dimensionality_offset(cls) -> int: @@ -1708,7 +1755,7 @@ the method signatures. @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DynamicCropBlockV1(WorkflowBlock): @@ -1739,21 +1786,21 @@ the method signatures. return results ``` - * in lines `31-33` manifest declares that block accepts batches of inputs + * in lines `29-31` manifest declares that block accepts batches of inputs - * in lines `35-37` manifest class declares output dimensionality + * in lines `33-35` manifest class declares output dimensionality offset - value `1` should be understood as adding `1` to dimensionality level - * in lines `57-68`, signature of input parameters reflects that the `run(...)` method + * in lines `55-66`, signature of input parameters reflects that the `run(...)` method runs against inputs of the same dimensionality and those inputs are provided in batches - * point out, that in line `72`, block eliminates empty images from further processing but + * point out, that in line `70`, block eliminates empty images from further processing but placing `None` instead of dictionatry with outputs. This would utilise the same Execution Engine behaviour that is used for conditional execution - datapoint will be eliminated from downstream processing (unless steps requesting empty inputs are present down the line). - * construction of the output, presented in lines `73-75` indicates two levels of nesting. + * construction of the output, presented in lines `71-73` indicates two levels of nesting. First of all, block operates on batches, so it is expected to return list of outputs, one output for each input batch element. Additionally, this output element for each input batch element turns out to be nested batch - hence for each input iage and prediction, block @@ -1765,7 +1812,7 @@ the method signatures. In this example, the block visualises crops predictions and creates tiles presenting all crops predictions in single output image. - ```{ .py linenums="1" hl_lines="31-33 35-37 54-55 68-69"} + ```{ .py linenums="1" hl_lines="29-31 33-35 52-53 66-67"} from typing import List, Literal, Type, Union import supervision as sv @@ -1778,9 +1825,7 @@ the method signatures. from inference.core.workflows.execution_engine.entities.types import ( IMAGE_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -1791,14 +1836,14 @@ the method signatures. class BlockManifest(WorkflowBlockManifest): type: Literal["my_plugin/tile_detections@v1"] - images_crops: Union[WorkflowImageSelector, StepOutputImageSelector] - crops_predictions: StepOutputSelector( + images_crops: Selector(kind=[IMAGE_KIND]) + crops_predictions: Selector( kind=[OBJECT_DETECTION_PREDICTION_KIND] ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> bool: + return ["images_crops", "crops_predictions"] @classmethod def get_output_dimensionality_offset(cls) -> int: @@ -1822,7 +1867,7 @@ the method signatures. images_crops: Batch[Batch[WorkflowImageData]], crops_predictions: Batch[Batch[sv.Detections]], ) -> BlockResult: - annotator = sv.BoundingBoxAnnotator() + annotator = sv.BoxAnnotator() visualisations = [] for image_crops, crop_predictions in zip(images_crops, crops_predictions): visualisations_batch_element = [] @@ -1837,19 +1882,19 @@ the method signatures. return visualisations ``` - * lines `31-33` manifest that block is expected to take batches as input + * lines `29-31` manifest that block is expected to take batches as input - * in lines `35-37` manifest class declares output dimensionality + * in lines `33-35` manifest class declares output dimensionality offset - value `-1` should be understood as decreasing dimensionality level by `1` - * in lines `54-55` you can see the impact of output dimensionality decrease + * in lines `52-53` you can see the impact of output dimensionality decrease and batch processing on the method signature. First "layer" of `Batch[]` is a side effect of the fact that manifest declared that block accepts batches of inputs. The second "layer" comes from output dimensionality decrease. Execution Engine wrapps up the dimension to be reduced into additional `Batch[]` container porvided in inputs, such that programmer is able to collect all nested batches elements that belong to specific top-level batch element. - * lines `68-69` illustrate how output is constructed - for each top-level batch element, block + * lines `66-67` illustrate how output is constructed - for each top-level batch element, block aggregates all crops and predictions and creates a single tile. As block accepts batches of inputs, this procedure end up with one tile for each top-level batch element - hence list of dictionaries is expected to be returned. @@ -1860,7 +1905,7 @@ the method signatures. crops of original image - result is to provide single detections with all partial ones being merged. - ```{ .py linenums="1" hl_lines="32-34 36-41 43-45 67-68 77-78"} + ```{ .py linenums="1" hl_lines="31-33 35-40 42-44 66-67 76-77"} from copy import deepcopy from typing import Dict, List, Literal, Optional, Type, Union @@ -1874,9 +1919,8 @@ the method signatures. ) from inference.core.workflows.execution_engine.entities.types import ( OBJECT_DETECTION_PREDICTION_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, + Selector, + IMAGE_KIND, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -1887,14 +1931,14 @@ the method signatures. class BlockManifest(WorkflowBlockManifest): type: Literal["my_plugin/stitch@v1"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] - images_predictions: StepOutputSelector( + images: Selector(kind=[IMAGE_KIND]) + images_predictions: Selector( kind=[OBJECT_DETECTION_PREDICTION_KIND], ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> bool: + return ["images", "images_predictions"] @classmethod def get_input_dimensionality_offsets(cls) -> Dict[str, int]: @@ -1941,26 +1985,26 @@ the method signatures. return result ``` - * lines `32-34` manifest that block is expected to take batches as input + * lines `31-33` manifest that block is expected to take batches as input - * in lines `36-41` manifest class declares input dimensionalities offset, indicating + * in lines `35-40` manifest class declares input dimensionalities offset, indicating `image` parameter being top-level and `image_predictions` being nested batch of predictions * whenever different input dimensionalities are declared, dimensionality reference property - must be pointed (see lines `43-45`) - this dimensionality level would be used to calculate + must be pointed (see lines `42-44`) - this dimensionality level would be used to calculate output dimensionality - in this particular case, we specify `image`. This choice has an implication in the expected format of result - in the chosen scenario we are supposed to return single dictionary for each element of `image` batch. If our choice is `image_predictions`, we would return list of dictionaries (of size equal to length of nested `image_predictions` batch) for each input `image` batch element. - * lines `67-68` present impact of dimensionality offsets specified in lines `36-41` as well as + * lines `66-67` present impact of dimensionality offsets specified in lines `35-40` as well as the declararion of batch processing from lines `32-34`. First "layer" of `Batch[]` container comes from the latter, nested `Batch[Batch[]]` for `images_predictions` comes from the definition of input dimensionality offset. It is clearly visible that `image_predictions` holds batch of predictions relevant for specific elements of `image` batch. - * as mentioned earlier, lines `77-78` construct output being single dictionary for each element of `image` + * as mentioned earlier, lines `76-77` construct output being single dictionary for each element of `image` batch @@ -1989,7 +2033,7 @@ that even if some elements are empty, the output lacks missing elements making i Batch, OutputDefinition, ) - from inference.core.workflows.execution_engine.entities.types import StepOutputSelector + from inference.core.workflows.execution_engine.entities.types import Selector from inference.core.workflows.prototypes.block import ( BlockResult, WorkflowBlock, @@ -1999,7 +2043,7 @@ that even if some elements are empty, the output lacks missing elements making i class BlockManifest(WorkflowBlockManifest): type: Literal["my_plugin/first_non_empty_or_default@v1"] - data: List[StepOutputSelector()] + data: List[Selector()] default: Any @classmethod @@ -2012,7 +2056,7 @@ that even if some elements are empty, the output lacks missing elements making i @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class FirstNonEmptyOrDefaultBlockV1(WorkflowBlock): @@ -2072,7 +2116,7 @@ Let's see how to request init parameters while defining block. Batch, OutputDefinition, ) - from inference.core.workflows.execution_engine.entities.types import StepOutputSelector + from inference.core.workflows.execution_engine.entities.types import Selector from inference.core.workflows.prototypes.block import ( BlockResult, WorkflowBlock, @@ -2082,7 +2126,7 @@ Let's see how to request init parameters while defining block. class BlockManifest(WorkflowBlockManifest): type: Literal["my_plugin/example@v1"] - data: List[StepOutputSelector()] + data: List[Selector()] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: diff --git a/docs/workflows/definitions.md b/docs/workflows/definitions.md index edb1cc799..ec2768aee 100644 --- a/docs/workflows/definitions.md +++ b/docs/workflows/definitions.md @@ -14,7 +14,7 @@ analyse it step by step. "version": "1.0", "inputs": [ { - "type": "InferenceImage", + "type": "WorkflowImage", "name": "image" }, { @@ -96,7 +96,7 @@ Our example workflow specifies two inputs: ```json [ { - "type": "InferenceImage", "name": "image" + "type": "WorkflowImage", "name": "image" }, { "type": "WorkflowParameter", "name": "model", "default_value": "yolov8n-640" @@ -105,9 +105,9 @@ Our example workflow specifies two inputs: ``` This entry in definition creates two placeholders that can be filled with data while running workflow. -The first placeholder is named `image` and is of type `InferenceImage`. This special input type is batch-oriented, +The first placeholder is named `image` and is of type `WorkflowImage`. This special input type is batch-oriented, meaning it can accept one or more images at runtime to be processed as a single batch. You can add multiple inputs -of the type `InferenceImage`, and it is expected that the data provided to these placeholders will contain +of the type `WorkflowImage`, and it is expected that the data provided to these placeholders will contain the same number of elements. Alternatively, you can mix inputs of sizes `N` and 1, where `N` represents the number of elements in the batch. @@ -119,6 +119,51 @@ elements, rather than batch of elements, each to be processed individually. More details about the nature of batch-oriented data processing in workflows can be found [here](/workflows/workflow_execution). +### Generic batch-oriented inputs + +Since Execution Engine `v1.3.0` (inference release `v0.27.0`), Workflows support +batch oriented inputs of any *[kind](/workflows/kinds/)* and +*[dimensionality](/workflows/workflow_execution/#steps-interactions-with-data)*. +This inputs are **not enforced for now**, but we expect that as the ecosystem grows, they will +be more and more useful. + +??? Tip "Defining generic batch-oriented inputs" + + If you wanted to replace the `WorkflowImage` input with generic batch-oriented input, + use the following construction: + + ```json + { + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "image", + "kind": ["image"] + } + ] + } + ``` + + Additionally, if your image is supposed to sit at higher *dimensionality level*, + add `dimensionality` property: + + ```{ .json linenums="1" hl_lines="7" } + { + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "image", + "kind": ["image"], + "dimensionality": 2 + } + ] + } + ``` + + This will alter the expected format of `image` data in Workflow run - + `dimensionality=2` enforces `image` to be nested batch of images - namely list + of list of images. + ## Steps diff --git a/docs/workflows/execution_engine_changelog.md b/docs/workflows/execution_engine_changelog.md index c3510700d..719ab865a 100644 --- a/docs/workflows/execution_engine_changelog.md +++ b/docs/workflows/execution_engine_changelog.md @@ -38,3 +38,266 @@ include a new `video_metadata` property. This property can be optionally set in a default value with reasonable defaults will be used. To simplify metadata manipulation within blocks, we have introduced two new class methods: `WorkflowImageData.copy_and_replace(...)` and `WorkflowImageData.create_crop(...)`. For more details, refer to the updated [`WoorkflowImageData` usage guide](/workflows/internal_data_types/#workflowimagedata). + + +## Execution Engine `v1.3.0` | inference `v0.27.0` + +* Introduced the change that let each kind have serializer and deserializer defined. The change decouples Workflows +plugins with Execution Engine and make it possible to integrate the ecosystem with external systems that +require data transfer through the wire. [Blocks bundling](/workflows/blocks_bundling/) page was updated to reflect +that change. + +* *Kinds* defined in `roboflow_core` plugin were provided with suitable serializers and deserializers + +* Workflows Compiler and Execution Engine were enhanced to **support batch-oriented inputs of +any *kind***, contrary to versions prior `v1.3.0`, which could only take `image` and `video_metadata` kinds +as batch-oriented inputs (as a result of unfortunate and not-needed coupling of kind to internal data +format introduced **at the level of Execution Engine**). As a result of the change: + + * **new input type was introduced:** `WorkflowBatchInput` should be used from now on to denote + batch-oriented inputs (and clearly separate them from `WorkflowParameters`). `WorkflowBatchInput` + let users define both *[kind](/workflows/kinds/)* of the data and it's + *[dimensionality](/workflows/workflow_execution/#steps-interactions-with-data)*. + New input type is effectively a superset of all previous batch-oriented inputs: `WorkflowImage` and + `WorkflowVideoMetadata`, which **remain supported**, but **will be removed in Execution Engine `v2`**. + We advise adjusting to the new input format, yet the requirement is not strict at the moment - as + Execution Engine requires now explicit definition of input data *kind* to select data deserializer + properly. This may not be the case in the future, as in most cases batch-oriented data *kind* may + be inferred by compiler (yet this feature is not implemented for now). + + * **new selector type annotation was introduced** - named simply `Selector(...)`. + `Selector(...)` is supposed to replace `StepOutputSelector`, `WorkflowImageSelector`, `StepOutputImageSelector`, + `WorkflowVideoMetadataSelector` and `WorkflowParameterSelector` in block manifests, + letting developers express that specific step manifest property is able to hold either selector of specific *kind*. + Mentioned old annotation types **should be assumed deprecated**, we advise to migrate into `Selector(...)`. + + * as a result of simplification in the selectors type annotations, the old selector will no + longer be providing the information on which parameter of blocks' `run(...)` method is + shipped by Execution Engine wrapped into [`Batch[X]` container](/workflows/internal_data_types/#batch). + Instead of old selectors type annotations and `block_manifest.accepts_batch_input()` method, + we propose the switch into two methods explicitly defining the parameters that are expected to + be fed with batch-oriented data (`block_manifest.get_parameters_accepting_batches()`) and + parameters capable of taking both *batches* and *scalar* values + (`block_manifest.get_parameters_accepting_batches_and_scalars()`). Return value of `block_manifest.accepts_batch_input()` + is built upon the results of two new methods. The change is **non-breaking**, as any existing block which + was capable of processing batches must have implemented `block_manifest.accepts_batch_input()` method returning + `True` and use appropriate selector type annotation which indicated batch-oriented data. + +* As a result of the changes, it is now possible to **split any arbitrary workflows into multiple ones executing +subsets of steps**, enabling building such tools as debuggers. + +!!! warning "Breaking change planned - Execution Engine `v2.0.0`" + + * `WorkflowImage` and `WorkflowVideoMetadata` inputs will be removed from Workflows ecosystem. + + * `StepOutputSelector, `WorkflowImageSelector`, `StepOutputImageSelector`, `WorkflowVideoMetadataSelector` + and `WorkflowParameterSelector` type annotations used in block manifests will be removed from Workflows ecosystem. + + +### Migration guide + +??? Hint "Kinds' serializers and deserializers" + + Creating your Workflows plugin you may introduce custom serializers and deserializers + for Workflows *kinds*. To achieve that end, simply place the following dictionaries + in the main module of the plugin (the same where you place `load_blocks(...)` function): + + ```python + from typing import Any + + def serialize_kind(value: Any) -> Any: + # place here the code that will be used to + # transform internal Workflows data representation into + # the external one (that can be sent through the wire in JSON, using + # default JSON encoder for Python). + pass + + + def deserialize_kind(parameter_name: str, value: Any) -> Any: + # place here the code that will be used to decode + # data sent through the wire into the Execution Engine + # and transform it into proper internal Workflows data representation + # which is understood by the blocks. + pass + + + KINDS_SERIALIZERS = { + "name_of_the_kind": serialize_kind, + } + KINDS_DESERIALIZERS = { + "name_of_the_kind": deserialize_kind, + } + ``` + +??? Hint "New type annotation for selectors - blocks without `Batch[X]` inputs" + + Blocks manifest may **optionally** be updated to use `Selector` in the following way: + + ```python + from typing import Union + from inference.core.workflows.prototypes.block import WorkflowBlockManifest + from inference.core.workflows.execution_engine.entities.types import ( + INSTANCE_SEGMENTATION_PREDICTION_KIND, + OBJECT_DETECTION_PREDICTION_KIND, + FLOAT_KIND, + WorkflowImageSelector, + StepOutputImageSelector, + StepOutputSelector, + WorkflowParameterSelector, + ) + + + class BlockManifest(WorkflowBlockManifest): + + reference_image: Union[WorkflowImageSelector, StepOutputImageSelector] + predictions: StepOutputSelector( + kind=[ + OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + ] + ) + confidence: WorkflowParameterSelector(kind=[FLOAT_KIND]) + ``` + + should just be changed into: + + ```{ .py linenums="1" hl_lines="7 12 13 19"} + from inference.core.workflows.prototypes.block import WorkflowBlockManifest + from inference.core.workflows.execution_engine.entities.types import ( + INSTANCE_SEGMENTATION_PREDICTION_KIND, + OBJECT_DETECTION_PREDICTION_KIND, + FLOAT_KIND, + IMAGE_KIND, + Selector, + ) + + + class BlockManifest(WorkflowBlockManifest): + reference_image: Selector(kind=[IMAGE_KIND]) + predictions: Selector( + kind=[ + OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + ] + ) + confidence: Selector(kind=[FLOAT_KIND]) + ``` + +??? Hint "New type annotation for selectors - blocks with `Batch[X]` inputs" + + Blocks manifest may **optionally** be updated to use `Selector` in the following way: + + ```python + from typing import Union + from inference.core.workflows.prototypes.block import WorkflowBlockManifest + from inference.core.workflows.execution_engine.entities.types import ( + INSTANCE_SEGMENTATION_PREDICTION_KIND, + OBJECT_DETECTION_PREDICTION_KIND, + FLOAT_KIND, + WorkflowImageSelector, + StepOutputImageSelector, + StepOutputSelector, + WorkflowParameterSelector, + ) + + + class BlockManifest(WorkflowBlockManifest): + + reference_image: Union[WorkflowImageSelector, StepOutputImageSelector] + predictions: StepOutputSelector( + kind=[ + OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + ] + ) + data: Dict[str, Union[StepOutputSelector(), WorkflowParameterSelector()]] + confidence: WorkflowParameterSelector(kind=[FLOAT_KIND]) + + @classmethod + def accepts_batch_input(cls) -> bool: + return True + ``` + + should be changed into: + + ```{ .py linenums="1" hl_lines="7 12 13 19 20 22-24 26-28"} + from inference.core.workflows.prototypes.block import WorkflowBlockManifest + from inference.core.workflows.execution_engine.entities.types import ( + INSTANCE_SEGMENTATION_PREDICTION_KIND, + OBJECT_DETECTION_PREDICTION_KIND, + FLOAT_KIND, + IMAGE_KIND, + Selector, + ) + + + class BlockManifest(WorkflowBlockManifest): + reference_image: Selector(kind=[IMAGE_KIND]) + predictions: Selector( + kind=[ + OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + ] + ) + data: Dict[str, Selector()] + confidence: Selector(kind=[FLOAT_KIND]) + + @classmethod + def get_parameters_accepting_batches(cls)W -> List[str]: + return ["predictions"] + + @classmethod + def get_parameters_accepting_batches_and_scalars(cls) -> List[str]: + return ["data"] + ``` + + Please point out that: + + * the `data` property in the original example was able to accept both **batches** of data + and **scalar** values due to selector of batch-orienetd data (`StepOutputSelector`) and + *scalar* data (`WorkflowParameterSelector`). Now the same is manifested by `Selector(...)` type + annotation and return value from `get_parameters_accepting_batches_and_scalars(...)` method. + + +??? Hint "New inputs in Workflows definitions" + + Anyone that used either `WorkflowImage` or `WorkflowVideoMetadata` inputs in their + Workflows definition may **optionally** migrate into `WorkflowBatchInput`. The transition + is illustrated below: + + ```json + { + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + {"type": "WorkflowVideoMetadata", "name": "video_metadata"} + ] + } + ``` + + should be changed into: + ```json + { + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "image", + "kind": ["image"] + }, + { + "type": "WorkflowBatchInput", + "name": "video_metadata", + "kind": ["video_metadata"] + } + ] + } + ``` + + **Leaving `kind` field empty may prevent some data - like images - from being deserialized properly.** + + + !!! Note + + If you do not like the way how data is serialized in `roboflow_core` plugin, + feel free to alter the serialization methods for *kinds*, simply registering + the function in your plugin and loading it to the Execution Engine - the + serializer/deserializer defined as the last one will be in use. diff --git a/docs/workflows/internal_data_types.md b/docs/workflows/internal_data_types.md index 8e4d5921f..d2494fd8d 100644 --- a/docs/workflows/internal_data_types.md +++ b/docs/workflows/internal_data_types.md @@ -284,6 +284,9 @@ def inspect_vide_metadata(video_metadata: VideoMetadata) -> None: # Field represents FPS value (if possible to be retrieved) (optional) print(video_metadata.fps) + # Field represents measured FPS of live stream (optional) + print(video_metadata.measured_fps) + # Field is a flag telling if frame comes from video file or stream. # If not possible to be determined - None print(video_metadata.comes_from_video_file) diff --git a/docs/workflows/kinds.md b/docs/workflows/kinds.md index 6be93488f..ed14bd8ab 100644 --- a/docs/workflows/kinds.md +++ b/docs/workflows/kinds.md @@ -1,19 +1,38 @@ -# Workflows kinds +# Kinds -In Workflows, some values cannot be defined when the Workflow Definition is created. To address this, the Execution -Engine supports selectors, which are references to step outputs or workflow inputs. To help the Execution Engine -understand what type of data will be provided once a reference is resolved, we use a simple type system known as -`kinds`. +In Workflows, some values can’t be set in advance and are only determined during execution. +This is similar to writing a function where you don’t know the exact input values upfront — they’re only +provided at runtime, either from user inputs or from other function outputs. -`Kinds` are used to represent the semantic meaning of the underlying data. When a step outputs data of a specific -`kind` and another step requires input of that same `kind`, the system assumes that the data will be compatible. -This reduces the need for extensive type-compatibility checks. +To manage this, Workflows use *selectors*, which act like references, pointing to data without containing it directly. + +!!! Example *selectors* + + Selectors might refer to a named input - for example input image - like `$inputs.image` + or predictions generated by a previous step - like `$steps.my_model.predictions` + +In the Workflows ecosystem, users focus on data purpose (e.g., “image”) without worrying about its exact format. +Meanwhile, developers building workflow blocks need precise data formats. **Kinds** serve both needs - +they simplify data handling for users while ensuring developers work with the correct data structure. + + +## What are the **Kinds**? + +**Kinds** is Workflows type system with each **kind** defining: + +* **name** - expressing **semantic meaning** of the underlying data - like `image` or `point`; + +* **Python data representation** - the data type and format that blocks creators should expect when handling +the data within blocks; + +* optional **serialized data representation** - defining what is the format of the kind that +external systems should use to integrate with Workflows ecosystem - when needed, custom kinds serializers +and deserializers are provided to ensure seamless translation; + +Using kinds streamlines compatibility: when a step outputs data of a certain *kind* and another step requires that +same *kind*, the workflow engine assumes they’ll be compatible, reducing the need for compatibility checks and +providing compile-time verification of Workflows definitions. -For example, we have different kinds to distinguish between predictions from `object detection` and -`instance segmentation` models, even though representation of those `kinds` is -[`sv.Detections(...)`](https://supervision.roboflow.com/latest/detection/core/). This distinction ensures that each -block that needs a segmentation mask clearly indicates this requirement, avoiding the need to repeatedly check -for the presence of a mask in the input. !!! Note @@ -33,41 +52,53 @@ for the presence of a mask in the input. never existed in the ecosystem and fixed all blocks from `roboflow_core` plugin. If there is anyone impacted by the change - here is the [migration guide](https://github.com/roboflow/inference/releases/tag/v0.18.0). + + This warning **will be removed end of Q1 2025**. +!!! Warning + + Support for proper serialization and deserialization of any arbitrary *kind* was + introduced in Execution Engine `v1.3.0` (released with inference `0.26.0`). Workflows + plugins created prior that change may be updated - see refreshed + [Blocks Bundling](/workflows/blocks_bundling/) page. + + This warning **will be removed end of Q1 2025**. + + ## Kinds declared in Roboflow plugins -* [`integer`](/workflows/kinds/integer): Integer value +* [`image`](/workflows/kinds/image): Image in workflows +* [`float`](/workflows/kinds/float): Float value +* [`numpy_array`](/workflows/kinds/numpy_array): Numpy array +* [`prediction_type`](/workflows/kinds/prediction_type): String value with type of prediction +* [`language_model_output`](/workflows/kinds/language_model_output): LLM / VLM output +* [`image_metadata`](/workflows/kinds/image_metadata): Dictionary with image metadata required by supervision +* [`keypoint_detection_prediction`](/workflows/kinds/keypoint_detection_prediction): Prediction with detected bounding boxes and detected keypoints in form of sv.Detections(...) object +* [`top_class`](/workflows/kinds/top_class): String value representing top class predicted by classification model +* [`video_metadata`](/workflows/kinds/video_metadata): Video image metadata +* [`qr_code_detection`](/workflows/kinds/qr_code_detection): Prediction with QR code detection +* [`contours`](/workflows/kinds/contours): List of numpy arrays where each array represents contour points * [`roboflow_model_id`](/workflows/kinds/roboflow_model_id): Roboflow model id * [`object_detection_prediction`](/workflows/kinds/object_detection_prediction): Prediction with detected bounding boxes in form of sv.Detections(...) object -* [`video_metadata`](/workflows/kinds/video_metadata): Video image metadata -* [`string`](/workflows/kinds/string): String value -* [`roboflow_api_key`](/workflows/kinds/roboflow_api_key): Roboflow API key -* [`detection`](/workflows/kinds/detection): Single element of detections-based prediction (like `object_detection_prediction`) +* [`roboflow_project`](/workflows/kinds/roboflow_project): Roboflow project name +* [`image_keypoints`](/workflows/kinds/image_keypoints): Image keypoints detected by classical Computer Vision method * [`list_of_values`](/workflows/kinds/list_of_values): List of values of any type -* [`instance_segmentation_prediction`](/workflows/kinds/instance_segmentation_prediction): Prediction with detected bounding boxes and segmentation masks in form of sv.Detections(...) object * [`float_zero_to_one`](/workflows/kinds/float_zero_to_one): `float` value in range `[0.0, 1.0]` -* [`image`](/workflows/kinds/image): Image in workflows -* [`image_metadata`](/workflows/kinds/image_metadata): Dictionary with image metadata required by supervision -* [`image_keypoints`](/workflows/kinds/image_keypoints): Image keypoints detected by classical Computer Vision method +* [`instance_segmentation_prediction`](/workflows/kinds/instance_segmentation_prediction): Prediction with detected bounding boxes and segmentation masks in form of sv.Detections(...) object +* [`rgb_color`](/workflows/kinds/rgb_color): RGB color +* [`boolean`](/workflows/kinds/boolean): Boolean flag * [`bar_code_detection`](/workflows/kinds/bar_code_detection): Prediction with barcode detection -* [`bytes`](/workflows/kinds/bytes): This kind represent bytes -* [`roboflow_project`](/workflows/kinds/roboflow_project): Roboflow project name -* [`dictionary`](/workflows/kinds/dictionary): Dictionary -* [`numpy_array`](/workflows/kinds/numpy_array): Numpy array -* [`qr_code_detection`](/workflows/kinds/qr_code_detection): Prediction with QR code detection * [`classification_prediction`](/workflows/kinds/classification_prediction): Predictions from classifier -* [`contours`](/workflows/kinds/contours): List of numpy arrays where each array represents contour points -* [`serialised_payloads`](/workflows/kinds/serialised_payloads): Serialised element that is usually accepted by sink -* [`prediction_type`](/workflows/kinds/prediction_type): String value with type of prediction -* [`zone`](/workflows/kinds/zone): Definition of polygon zone -* [`keypoint_detection_prediction`](/workflows/kinds/keypoint_detection_prediction): Prediction with detected bounding boxes and detected keypoints in form of sv.Detections(...) object -* [`boolean`](/workflows/kinds/boolean): Boolean flag -* [`float`](/workflows/kinds/float): Float value -* [`point`](/workflows/kinds/point): Single point in 2D -* [`top_class`](/workflows/kinds/top_class): String value representing top class predicted by classification model -* [`language_model_output`](/workflows/kinds/language_model_output): LLM / VLM output +* [`string`](/workflows/kinds/string): String value * [`parent_id`](/workflows/kinds/parent_id): Identifier of parent for step output +* [`point`](/workflows/kinds/point): Single point in 2D +* [`bytes`](/workflows/kinds/bytes): This kind represent bytes +* [`serialised_payloads`](/workflows/kinds/serialised_payloads): Serialised element that is usually accepted by sink +* [`dictionary`](/workflows/kinds/dictionary): Dictionary * [`*`](/workflows/kinds/*): Equivalent of any element -* [`rgb_color`](/workflows/kinds/rgb_color): RGB color +* [`detection`](/workflows/kinds/detection): Single element of detections-based prediction (like `object_detection_prediction`) +* [`integer`](/workflows/kinds/integer): Integer value +* [`zone`](/workflows/kinds/zone): Definition of polygon zone +* [`roboflow_api_key`](/workflows/kinds/roboflow_api_key): Roboflow API key diff --git a/docs/workflows/video_processing/overview.md b/docs/workflows/video_processing/overview.md index 20d46fa2d..45f8a65ac 100644 --- a/docs/workflows/video_processing/overview.md +++ b/docs/workflows/video_processing/overview.md @@ -5,7 +5,7 @@ video-specific blocks (e.g., the ByteTracker block) and continue to dedicate eff their performance and robustness. The current state of this work is as follows: * We've introduced the `WorkflowVideoMetadata` input to store metadata related to video frames, -including FPS, timestamp, video source identifier, and file/stream flags. While this may not be the final approach +including declared FPS, measured FPS, timestamp, video source identifier, and file/stream flags. While this may not be the final approach for handling video metadata, it allows us to build stateful video-processing blocks at this stage. If your Workflow includes any blocks requiring input of kind `video_metadata`, you must define this input in your Workflow. The metadata functions as a batch-oriented parameter, treated by the Execution Engine in the same diff --git a/docs/workflows/workflow_execution.md b/docs/workflows/workflow_execution.md index e7ee00fed..141daefa3 100644 --- a/docs/workflows/workflow_execution.md +++ b/docs/workflows/workflow_execution.md @@ -53,17 +53,21 @@ actual data values. It simply tells the Execution Engine how to direct and handl Input data in a Workflow can be divided into two types: -- Data to be processed: This can be submitted as a batch of data points. +- Batch-Oriented Data to be processed: Main data to be processed, which you expect to derive results +from (for instance: making inference with your model) -- Parameters: These are single values used for specific settings or configurations. +- Scalars: These are single values used for specific settings or configurations. -To clarify the difference, consider this simple Python function: +Thinking about standard data processing, like the one presented below, you may find the distinction +between scalars and batch-oriented data artificial. ```python def is_even(number: int) -> bool: return number % 2 == 0 ``` -You use this function like this, providing one number at a time: + +You can easily submit different values as `number` parameter and do not bother associating the +parameter into one of the two categories. ```python is_even(number=1) @@ -71,14 +75,15 @@ is_even(number=2) is_even(number=3) ``` -The situation becomes more complex with machine learning models. Unlike a simple function like `is_even(...)`, +The situation becomes more complicated with machine learning models. Unlike a simple function like `is_even(...)`, which processes one number at a time, ML models often handle multiple pieces of data at once. For example, instead of providing just one image to a classification model, you can usually submit a list of images and -receive predictions for all of them at once. +receive predictions for all of them at once performing **the same operation** for each image. This is different from our `is_even(...)` function, which would need to be called separately for each number to get a list of results. The difference comes from how ML models work, especially how -GPUs process data - applying the same operation to many pieces of data simultaneously. +GPUs process data - applying the same operation to many pieces of data simultaneously, executing +[Single Instruction Multiple Data](https://en.wikipedia.org/wiki/Single_instruction,_multiple_data) operations.
@@ -90,10 +95,12 @@ for number in [1, 2, 3, 4]: results.append(is_even(number)) ``` -In Workflows, similar methods are used to handle non-batch-oriented steps facing batch input data. But what if -step expects batch-oriented data and is given singular data point? Let's look at inference process from example -classification model: +In Workflows, usually **you do not need to worry** about broadcasting the operations into batches of data - +Execution Engine is doing that for you behind the scenes, but once you understood the role of *batch-oriented* +data, let's think if all data can be represented as batches. +Standard way of making predictions from classification model is be illustrated with the following +pseudo-code: ```python images = [PIL.Image(...), PIL.Image(...), PIL.Image(...), PIL.Image(...)] model = MyClassificationModel() @@ -101,38 +108,44 @@ model = MyClassificationModel() predictions = model.infer(images=images, confidence_threshold=0.5) ``` -As you may imagine, this code has chance to run correctly, as there is substantial difference in meaning of -`images` and `confidence_threshold` parameter. Former is batch of data to apply single operation (prediction -from a model) and the latter is parameter influencing the processing for all elements in the batch. Virtually, -`confidence_threshold` gets propagated (broadcast) at each element of `images` list with the same value, -as if `confidence_threshold` was the following list: `[0.5, 0.5, 0.5, 0.5]`. +You can probably spot the difference between `images` and `confidence_threshold`. +Former is batch of data to apply single operation (prediction from a model) and the latter is parameter +influencing the processing for all elements in the batch and this type of data we call **scalars**. + +!!! Tip "Nature of *batches* and *scalars*" + + What we call *scalar* in Workflows ecosystem is not 100% equivalent to the mathematical + term which is usually associated to "a single value", but in Workflows we prefer slightly different + definition. -As mentioned earlier, Workflow inputs can be of two types: + In the Workflows ecosystem, a *scalar* is a piece of data that stays constant, regardless of how many + elements are processed. There is nothing that prevents from having a list of objects as a *scalar* value. + For example, if you have a list of input images and a fixed list of reference images, + the reference images remain unchanged as you process each input. Thus, the reference images are considered + *scalar* data, while the list of input images is *batch-oriented*. -- `WorkflowImage`: This is similar to the images parameter in our example. +To illustrate the distinction, Workflow definitions hold inputs of the two categories: + +- **Scalar inputs** - like `WorkflowParameter` + +- **Batch inputs** - like `WorkflowImage`, `WorkflowVideoMetadata` or `WorkflowBatchInput` -- `WorkflowParameters`: This works like the confidence_threshold. When you provide a single image as a `WorkflowImage` input, it is automatically expanded to form a batch. If your Workflow definition includes multiple `WorkflowImage` placeholders, the actual data you provide for execution must have the same batch size for all these inputs. The only exception is when you submit a single image; it will be broadcast to fit the batch size requirements of other inputs. -Currently, `WorkflowImage` is the only type of batch-oriented input you can use in Workflows. -This was introduced because the ecosystem started in the Computer Vision field, where images are a key data type. -However, as the field evolves and expands to include multi-modal models (LMMs) and other types of data, -you can expect additional batch-oriented data types to be introduced in the future. - ## Steps interactions with data If we asked you about the nature of step outputs in these scenarios: -- **A**: The step receives non-batch-oriented parameters as input. +- **A**: The step receives only scalar parameters as input. - **B**: The step receives batch-oriented data as input. -- **C**: The step receives both non-batch-oriented parameters and batch-oriented data as input. +- **C**: The step receives both scalar parameters and batch-oriented data as input. You would likely say: @@ -141,8 +154,7 @@ You would likely say: - In options B and C, the output will be a batch. In option C, the non-batch-oriented parameters will be broadcast to match the batch size of the data. -And you’d be correct. If you understand that, you probably only have two more concepts to understand before -you can comfortably say you understand everything needed to successfully build and run complex Workflows. +And you’d be correct. Knowing that, you only have two more concepts to understand to become Workflows expert. Let’s say you want to create a Workflow with these steps: @@ -159,7 +171,7 @@ Here’s what happens with the data in the cropping step: 2. The object detection model finds a different number of objects in each image. -3. The cropping step then creates new images for each detected object, resulting in a new batch of images +3. The cropping step then creates new image for each detected object, resulting in a new batch of images for each original image. So, you end up with a nested list of images, with sizes like `[(k[1], ), (k[2], ), ... (k[n])]`, where each `k[i]` diff --git a/docs/workflows/workflows_compiler.md b/docs/workflows/workflows_compiler.md index 3e0fb6145..4d2bbf06d 100644 --- a/docs/workflows/workflows_compiler.md +++ b/docs/workflows/workflows_compiler.md @@ -232,6 +232,37 @@ is a batch of data - all batch elements are affected. * **The flow-control step operates on batch-oriented inputs with compatible lineage** - here, the flow-control step can decide separately for each element in the batch which ones will proceed and which ones will be stopped. +#### Batch-orientation compatibility + +As it was outlined, Workflows define **batch-oriented data** and **scalars**. +From [the description of the nature of data in Workflows](/workflows/workflow_execution/#what-is-the-data), +you can also conclude that operations which are executed against batch-oriented data +have two almost equivalent ways of running: + +* **all-at-once:** taking whole batches of data and processing them + +* **one-by-one:** looping over batch elements and getting results sequentially + +Since the default way for Workflow blocks to deal with the batches is to consume them element-by-element, +**there is no real difference** between **batch-oriented data** and **scalars** +in such case. Execution Engine simply unpacks scalars from batches and pass them to each step. + +The process may complicate when block accepts batch input. You will learn the +details in [blocks development guide](/workflows/create_workflow_block/), but +block is required to denote each input that must be provided *batch-wise* and all inputs +which can be fed with both batch-oriented data and scalars at the same time (which is much +less common case). In such cases, *lineage* is used to deduce if the actual data fed into +every step input is *batch* or *scalar*. When violation is detected (for instance *scalar* is provided for input +that requires batches or vice versa) - the error is raised. + + +!!! Note "Potential future improvements" + + At this moment, we are not sure if the behaviour described above is limiting the potential of + Workflows ecosystem. If you see that your Workflows cannot run due to the errors + being result of described mechanism - please let us know in + [GitHub issues](https://github.com/roboflow/inference/issues). + ## Initializing Workflow steps from blocks diff --git a/examples/notebooks/inference_sdk.ipynb b/examples/notebooks/inference_sdk.ipynb index f54cba595..d02baecb7 100644 --- a/examples/notebooks/inference_sdk.ipynb +++ b/examples/notebooks/inference_sdk.ipynb @@ -151,7 +151,7 @@ "detections = sv.Detections.from_inference(results)\n", "\n", "#Initialize annotators\n", - "bounding_box_annotator = sv.BoundingBoxAnnotator()\n", + "bounding_box_annotator = sv.BoxAnnotator()\n", "label_annotator = sv.LabelAnnotator()\n", "\n", "#Get class labels from inference results\n", diff --git a/examples/notebooks/quickstart.ipynb b/examples/notebooks/quickstart.ipynb index f78ea6154..d2b6d77b1 100644 --- a/examples/notebooks/quickstart.ipynb +++ b/examples/notebooks/quickstart.ipynb @@ -170,7 +170,7 @@ "detections = sv.Detections.from_inference(result.dict(by_alias=True, exclude_none=True))\n", "\n", "#Initialize annotators\n", - "bounding_box_annotator = sv.BoundingBoxAnnotator()\n", + "bounding_box_annotator = sv.BoxAnnotator()\n", "label_annotator = sv.LabelAnnotator()\n", "\n", "#Get class labels from inference results\n", diff --git a/examples/notebooks/workflows.ipynb b/examples/notebooks/workflows.ipynb index 11003df64..61bc0c62e 100644 --- a/examples/notebooks/workflows.ipynb +++ b/examples/notebooks/workflows.ipynb @@ -791,7 +791,7 @@ } ], "source": [ - "annotator = sv.BoundingBoxAnnotator(thickness=20)\n", + "annotator = sv.BoxAnnotator(thickness=20)\n", "detections = sv.Detections.from_inference(detection_coco_and_plates[\"predictions\"])\n", "plt.imshow(annotator.annotate(multiple_cars_image_2.copy(), detections)[:, :, ::-1])\n", "plt.show()" diff --git a/inference/core/entities/requests/inference.py b/inference/core/entities/requests/inference.py index 3a2107617..312a3baab 100644 --- a/inference/core/entities/requests/inference.py +++ b/inference/core/entities/requests/inference.py @@ -22,6 +22,7 @@ def __init__(self, **kwargs): model_config = ConfigDict(protected_namespaces=()) id: str api_key: Optional[str] = ApiKey + usage_billable: bool = True start: Optional[float] = None source: Optional[str] = None source_info: Optional[str] = None diff --git a/inference/core/interfaces/camera/entities.py b/inference/core/interfaces/camera/entities.py index 6e81e163e..218240d9c 100644 --- a/inference/core/interfaces/camera/entities.py +++ b/inference/core/interfaces/camera/entities.py @@ -55,14 +55,17 @@ class VideoFrame: frame_timestamp (FrameTimestamp): The timestamp when the frame was captured. source_id (int): The index of the video_reference element which was passed to InferencePipeline for this frame (useful when multiple streams are passed to InferencePipeline). - fps (Optional[float]): FPS of source (if possible to be acquired) + fps (Optional[float]): declared FPS of source (if possible to be acquired) + measured_fps (Optional[float]): measured FPS of live stream comes_from_video_file (Optional[bool]): flag to determine if frame comes from video file """ image: np.ndarray frame_id: FrameID frame_timestamp: FrameTimestamp + # TODO: in next major version of inference replace `fps` with `declared_fps` fps: Optional[float] = None + measured_fps: Optional[float] = None source_id: Optional[int] = None comes_from_video_file: Optional[bool] = None diff --git a/inference/core/interfaces/camera/video_source.py b/inference/core/interfaces/camera/video_source.py index 97bf2035d..b73a7211e 100644 --- a/inference/core/interfaces/camera/video_source.py +++ b/inference/core/interfaces/camera/video_source.py @@ -862,11 +862,19 @@ def consume_frame( }, status_update_handlers=self._status_update_handlers, ) + measured_source_fps = declared_source_fps + if not is_source_video_file: + if hasattr(self._stream_consumption_pace_monitor, "fps"): + measured_source_fps = self._stream_consumption_pace_monitor.fps + else: + measured_source_fps = self._stream_consumption_pace_monitor() + if self._video_fps_should_be_sub_sampled(): return True return self._consume_stream_frame( video=video, declared_source_fps=declared_source_fps, + measured_source_fps=measured_source_fps, is_source_video_file=is_source_video_file, frame_timestamp=frame_timestamp, buffer=buffer, @@ -912,6 +920,7 @@ def _consume_stream_frame( self, video: VideoFrameProducer, declared_source_fps: Optional[float], + measured_source_fps: Optional[float], is_source_video_file: Optional[bool], frame_timestamp: datetime, buffer: Queue, @@ -954,7 +963,8 @@ def _consume_stream_frame( buffer=buffer, decoding_pace_monitor=self._decoding_pace_monitor, source_id=source_id, - fps=declared_source_fps, + declared_source_fps=declared_source_fps, + measured_source_fps=measured_source_fps, comes_from_video_file=is_source_video_file, ) if self._buffer_filling_strategy in DROP_OLDEST_STRATEGIES: @@ -1153,7 +1163,8 @@ def decode_video_frame_to_buffer( buffer: Queue, decoding_pace_monitor: sv.FPSMonitor, source_id: Optional[int], - fps: Optional[float] = None, + declared_source_fps: Optional[float] = None, + measured_source_fps: Optional[float] = None, comes_from_video_file: Optional[bool] = None, ) -> bool: success, image = video.retrieve() @@ -1164,7 +1175,8 @@ def decode_video_frame_to_buffer( image=image, frame_id=frame_id, frame_timestamp=frame_timestamp, - fps=fps, + fps=declared_source_fps, + measured_fps=measured_source_fps, source_id=source_id, comes_from_video_file=comes_from_video_file, ) diff --git a/inference/core/interfaces/http/handlers/workflows.py b/inference/core/interfaces/http/handlers/workflows.py index ad9864576..e71c0d19b 100644 --- a/inference/core/interfaces/http/handlers/workflows.py +++ b/inference/core/interfaces/http/handlers/workflows.py @@ -137,3 +137,21 @@ def get_unique_kinds( for output_field_kinds in output_definition.values(): all_kinds.update(output_field_kinds) return all_kinds + + +def filter_out_unwanted_workflow_outputs( + workflow_results: List[dict], + excluded_fields: Optional[List[str]], +) -> List[dict]: + if not excluded_fields: + return workflow_results + excluded_fields = set(excluded_fields) + filtered_results = [] + for result_element in workflow_results: + filtered_result = {} + for key, value in result_element.items(): + if key in excluded_fields: + continue + filtered_result[key] = value + filtered_results.append(filtered_result) + return filtered_results diff --git a/inference/core/interfaces/http/http_api.py b/inference/core/interfaces/http/http_api.py index 04d0fd089..2ba5dd48f 100644 --- a/inference/core/interfaces/http/http_api.py +++ b/inference/core/interfaces/http/http_api.py @@ -7,7 +7,7 @@ import asgi_correlation_id import uvicorn -from fastapi import BackgroundTasks, Depends, FastAPI, Path, Query, Request +from fastapi import BackgroundTasks, FastAPI, Path, Query, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, RedirectResponse, Response from fastapi.staticfiles import StaticFiles @@ -155,14 +155,12 @@ ) from inference.core.interfaces.base import BaseInterface from inference.core.interfaces.http.handlers.workflows import ( + filter_out_unwanted_workflow_outputs, handle_describe_workflows_blocks_request, handle_describe_workflows_interface, ) from inference.core.interfaces.http.middlewares.gzip import gzip_response_if_requested -from inference.core.interfaces.http.orjson_utils import ( - orjson_response, - serialise_workflow_result, -) +from inference.core.interfaces.http.orjson_utils import orjson_response from inference.core.interfaces.stream_manager.api.entities import ( CommandResponse, ConsumePipelineResponse, @@ -723,13 +721,16 @@ def process_workflow_inference_request( prevent_local_images_loading=True, profiler=profiler, ) - result = execution_engine.run(runtime_parameters=workflow_request.inputs) + workflow_results = execution_engine.run( + runtime_parameters=workflow_request.inputs, + serialize_results=True, + ) with profiler.profile_execution_phase( - name="workflow_results_serialisation", + name="workflow_results_filtering", categories=["inference_package_operation"], ): - outputs = serialise_workflow_result( - result=result, + outputs = filter_out_unwanted_workflow_outputs( + workflow_results=workflow_results, excluded_fields=workflow_request.excluded_fields, ) profiler_trace = profiler.export_trace() @@ -2270,6 +2271,7 @@ async def legacy_infer_from_request( raise MissingServiceSecretError( "Service secret is required to disable inference usage tracking" ) + logger.info("Not counting inference for usage") else: request_model_id = model_id logger.debug( diff --git a/inference/core/interfaces/http/orjson_utils.py b/inference/core/interfaces/http/orjson_utils.py index 27ecb17d7..aa91baa8a 100644 --- a/inference/core/interfaces/http/orjson_utils.py +++ b/inference/core/interfaces/http/orjson_utils.py @@ -2,7 +2,6 @@ from typing import Any, Dict, List, Optional, Union import orjson -import supervision as sv from fastapi.responses import ORJSONResponse from pydantic import BaseModel @@ -10,10 +9,8 @@ from inference.core.utils.function import deprecated from inference.core.utils.image_utils import ImageType from inference.core.workflows.core_steps.common.serializers import ( - serialise_image, - serialise_sv_detections, + serialize_wildcard_kind, ) -from inference.core.workflows.execution_engine.entities.base import WorkflowImageData class ORJSONResponseBytes(ORJSONResponse): @@ -44,6 +41,11 @@ def orjson_response( return ORJSONResponseBytes(content=content) +@deprecated( + reason="Function serialise_workflow_result(...) will be removed from `inference` end of Q1 2025. " + "Workflows ecosystem shifted towards internal serialization - see Workflows docs: " + "https://inference.roboflow.com/workflows/about/" +) def serialise_workflow_result( result: List[Dict[str, Any]], excluded_fields: Optional[List[str]] = None, @@ -57,6 +59,11 @@ def serialise_workflow_result( ] +@deprecated( + reason="Function serialise_single_workflow_result_element(...) will be removed from `inference` end of Q1 2025. " + "Workflows ecosystem shifted towards internal serialization - see Workflows docs: " + "https://inference.roboflow.com/workflows/about/" +) def serialise_single_workflow_result_element( result_element: Dict[str, Any], excluded_fields: Optional[List[str]] = None, @@ -68,45 +75,7 @@ def serialise_single_workflow_result_element( for key, value in result_element.items(): if key in excluded_fields: continue - if isinstance(value, WorkflowImageData): - value = serialise_image(image=value) - elif isinstance(value, dict): - value = serialise_dict(elements=value) - elif isinstance(value, list): - value = serialise_list(elements=value) - elif isinstance(value, sv.Detections): - value = serialise_sv_detections(detections=value) - serialised_result[key] = value - return serialised_result - - -def serialise_list(elements: List[Any]) -> List[Any]: - result = [] - for element in elements: - if isinstance(element, WorkflowImageData): - element = serialise_image(image=element) - elif isinstance(element, dict): - element = serialise_dict(elements=element) - elif isinstance(element, list): - element = serialise_list(elements=element) - elif isinstance(element, sv.Detections): - element = serialise_sv_detections(detections=element) - result.append(element) - return result - - -def serialise_dict(elements: Dict[str, Any]) -> Dict[str, Any]: - serialised_result = {} - for key, value in elements.items(): - if isinstance(value, WorkflowImageData): - value = serialise_image(image=value) - elif isinstance(value, dict): - value = serialise_dict(elements=value) - elif isinstance(value, list): - value = serialise_list(elements=value) - elif isinstance(value, sv.Detections): - value = serialise_sv_detections(detections=value) - serialised_result[key] = value + serialised_result[key] = serialize_wildcard_kind(value=value) return serialised_result diff --git a/inference/core/interfaces/stream/model_handlers/roboflow_models.py b/inference/core/interfaces/stream/model_handlers/roboflow_models.py index 3145b135b..cb2d995a9 100644 --- a/inference/core/interfaces/stream/model_handlers/roboflow_models.py +++ b/inference/core/interfaces/stream/model_handlers/roboflow_models.py @@ -1,4 +1,4 @@ -from typing import Any, List +from typing import List from inference.core.interfaces.camera.entities import VideoFrame from inference.core.interfaces.stream.entities import ModelConfig @@ -12,10 +12,16 @@ def default_process_frame( inference_config: ModelConfig, ) -> List[dict]: postprocessing_args = inference_config.to_postprocessing_params() + # TODO: handle batch input in usage + fps = video_frame[0].fps + if video_frame[0].measured_fps: + fps = video_frame[0].measured_fps + if not fps: + fps = 0 predictions = wrap_in_list( model.infer( [f.image for f in video_frame], - usage_fps=video_frame[0].fps, + usage_fps=fps, usage_api_key=model.api_key, **postprocessing_args, ) diff --git a/inference/core/interfaces/stream/model_handlers/workflows.py b/inference/core/interfaces/stream/model_handlers/workflows.py index e8c798541..c1da0c440 100644 --- a/inference/core/interfaces/stream/model_handlers/workflows.py +++ b/inference/core/interfaces/stream/model_handlers/workflows.py @@ -19,6 +19,8 @@ def run_workflow( workflows_parameters = {} # TODO: pass fps reflecting each stream to workflows_parameters fps = video_frames[0].fps + if video_frames[0].measured_fps: + fps = video_frames[0].measured_fps if fps is None: # for FPS reporting we expect 0 when FPS cannot be determined fps = 0 @@ -32,6 +34,7 @@ def run_workflow( frame_number=video_frame.frame_id, frame_timestamp=video_frame.frame_timestamp, fps=video_frame.fps, + measured_fps=video_frame.measured_fps, comes_from_video_file=video_frame.comes_from_video_file, ) for video_frame in video_frames diff --git a/inference/core/interfaces/stream/sinks.py b/inference/core/interfaces/stream/sinks.py index cd6d2a4bb..ca449fde8 100644 --- a/inference/core/interfaces/stream/sinks.py +++ b/inference/core/interfaces/stream/sinks.py @@ -18,7 +18,7 @@ from inference.core.utils.drawing import create_tiles from inference.core.utils.preprocess import letterbox_image -DEFAULT_BBOX_ANNOTATOR = sv.BoundingBoxAnnotator() +DEFAULT_BBOX_ANNOTATOR = sv.BoxAnnotator() DEFAULT_LABEL_ANNOTATOR = sv.LabelAnnotator() DEFAULT_FPS_MONITOR = sv.FPSMonitor() @@ -50,8 +50,8 @@ def render_boxes( ) -> None: """ Helper tool to render object detection predictions on top of video frame. It is designed - to be used with `InferencePipeline`, as sink for predictions. By default it uses - standard `sv.BoundingBoxAnnotator()` chained with `sv.LabelAnnotator()` + to be used with `InferencePipeline`, as sink for predictions. By default, it uses + standard `sv.BoxAnnotator()` chained with `sv.LabelAnnotator()` to draw bounding boxes and resizes prediction to 1280x720 (keeping aspect ratio and adding black padding). One may configure default behaviour, for instance to display latency and throughput statistics. In batch mode it will display tiles of frames and overlay predictions. @@ -70,7 +70,7 @@ def render_boxes( by `VideoSource` or list of frames from (it is possible for empty batch frames at corresponding positions to `predictions` list). Order is expected to match with `predictions` annotator (Union[BaseAnnotator, List[BaseAnnotator]]): instance of class inheriting from supervision BaseAnnotator - or list of such instances. If nothing is passed chain of `sv.BoundingBoxAnnotator()` and `sv.LabelAnnotator()` is used. + or list of such instances. If nothing is passed chain of `sv.BoxAnnotator()` and `sv.LabelAnnotator()` is used. display_size (Tuple[int, int]): tuple in format (width, height) to resize visualisation output fps_monitor (Optional[sv.FPSMonitor]): FPS monitor used to monitor throughput display_statistics (bool): Flag to decide if throughput and latency can be displayed in the result image, @@ -424,7 +424,7 @@ def init( Args: video_file_name (str): name of the video file to save predictions annotator (Union[BaseAnnotator, List[BaseAnnotator]]): instance of class inheriting from supervision BaseAnnotator - or list of such instances. If nothing is passed chain of `sv.BoundingBoxAnnotator()` and `sv.LabelAnnotator()` is used. + or list of such instances. If nothing is passed chain of `sv.BoxAnnotator()` and `sv.LabelAnnotator()` is used. display_size (Tuple[int, int]): tuple in format (width, height) to resize visualisation output. Should be set to the same value as `display_size` for InferencePipeline with single video source, otherwise it represents the size of single visualisation tile (whole tiles mosaic will be scaled to diff --git a/inference/core/interfaces/stream_manager/manager_app/entities.py b/inference/core/interfaces/stream_manager/manager_app/entities.py index bbf4f7270..8b017c0d9 100644 --- a/inference/core/interfaces/stream_manager/manager_app/entities.py +++ b/inference/core/interfaces/stream_manager/manager_app/entities.py @@ -91,8 +91,15 @@ class WebRTCOffer(BaseModel): sdp: str +class WebRTCTURNConfig(BaseModel): + urls: str + username: str + credential: str + + class InitialiseWebRTCPipelinePayload(InitialisePipelinePayload): webrtc_offer: WebRTCOffer + webrtc_turn_config: WebRTCTURNConfig stream_output: Optional[List[str]] = Field(default_factory=list) data_output: Optional[List[str]] = Field(default_factory=list) webrtc_peer_timeout: float = 1 diff --git a/inference/core/interfaces/stream_manager/manager_app/inference_pipeline_manager.py b/inference/core/interfaces/stream_manager/manager_app/inference_pipeline_manager.py index cfb7cb552..e037a0718 100644 --- a/inference/core/interfaces/stream_manager/manager_app/inference_pipeline_manager.py +++ b/inference/core/interfaces/stream_manager/manager_app/inference_pipeline_manager.py @@ -242,6 +242,7 @@ def start_loop(loop: asyncio.AbstractEventLoop): t.start() webrtc_offer = parsed_payload.webrtc_offer + webrtc_turn_config = parsed_payload.webrtc_turn_config webcam_fps = parsed_payload.webcam_fps to_inference_queue = SyncAsyncQueue(loop=loop) from_inference_queue = SyncAsyncQueue(loop=loop) @@ -251,6 +252,7 @@ def start_loop(loop: asyncio.AbstractEventLoop): future = asyncio.run_coroutine_threadsafe( init_rtc_peer_connection( webrtc_offer=webrtc_offer, + webrtc_turn_config=webrtc_turn_config, to_inference_queue=to_inference_queue, from_inference_queue=from_inference_queue, webrtc_peer_timeout=parsed_payload.webrtc_peer_timeout, diff --git a/inference/core/interfaces/stream_manager/manager_app/webrtc.py b/inference/core/interfaces/stream_manager/manager_app/webrtc.py index 24431955b..dd5288980 100644 --- a/inference/core/interfaces/stream_manager/manager_app/webrtc.py +++ b/inference/core/interfaces/stream_manager/manager_app/webrtc.py @@ -5,7 +5,13 @@ from typing import Dict, Optional, Tuple import numpy as np -from aiortc import RTCPeerConnection, RTCSessionDescription, VideoStreamTrack +from aiortc import ( + RTCConfiguration, + RTCIceServer, + RTCPeerConnection, + RTCSessionDescription, + VideoStreamTrack, +) from aiortc.contrib.media import MediaRelay from aiortc.mediastreams import MediaStreamError from aiortc.rtcrtpreceiver import RemoteStreamTrack @@ -16,7 +22,10 @@ SourceProperties, VideoFrameProducer, ) -from inference.core.interfaces.stream_manager.manager_app.entities import WebRTCOffer +from inference.core.interfaces.stream_manager.manager_app.entities import ( + WebRTCOffer, + WebRTCTURNConfig, +) from inference.core.utils.async_utils import Queue as SyncAsyncQueue from inference.core.utils.function import experimental @@ -203,6 +212,7 @@ def __init__(self, video_transform_track: VideoTransformTrack, *args, **kwargs): async def init_rtc_peer_connection( webrtc_offer: WebRTCOffer, + webrtc_turn_config: WebRTCTURNConfig, to_inference_queue: "SyncAsyncQueue[VideoFrame]", from_inference_queue: "SyncAsyncQueue[np.ndarray]", webrtc_peer_timeout: float, @@ -218,8 +228,14 @@ async def init_rtc_peer_connection( webcam_fps=webcam_fps, ) + turn_server = RTCIceServer( + urls=[webrtc_turn_config.urls], + username=webrtc_turn_config.username, + credential=webrtc_turn_config.credential, + ) peer_connection = RTCPeerConnectionWithFPS( - video_transform_track=video_transform_track + video_transform_track=video_transform_track, + configuration=RTCConfiguration(iceServers=[turn_server]), ) relay = MediaRelay() diff --git a/inference/core/version.py b/inference/core/version.py index c14735062..4a6e856f0 100644 --- a/inference/core/version.py +++ b/inference/core/version.py @@ -1,4 +1,4 @@ -__version__ = "0.26.0" +__version__ = "0.26.1" if __name__ == "__main__": diff --git a/inference/core/workflows/core_steps/analytics/data_aggregator/v1.py b/inference/core/workflows/core_steps/analytics/data_aggregator/v1.py index 5f467e249..e4bb6adb4 100644 --- a/inference/core/workflows/core_steps/analytics/data_aggregator/v1.py +++ b/inference/core/workflows/core_steps/analytics/data_aggregator/v1.py @@ -18,9 +18,7 @@ FLOAT_KIND, INTEGER_KIND, LIST_OF_VALUES_KIND, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -192,10 +190,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/data_aggregator@v1"] - data: Dict[ - str, - Union[WorkflowImageSelector, WorkflowParameterSelector(), StepOutputSelector()], - ] = Field( + data: Dict[str, Selector()] = Field( description="References data to be used to construct each and every column", examples=[ { @@ -326,7 +321,7 @@ def get_actual_outputs(self) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" INTERVAL_UNIT_TO_SECONDS = { diff --git a/inference/core/workflows/core_steps/analytics/line_counter/v1.py b/inference/core/workflows/core_steps/analytics/line_counter/v1.py index e1892119c..ba739345f 100644 --- a/inference/core/workflows/core_steps/analytics/line_counter/v1.py +++ b/inference/core/workflows/core_steps/analytics/line_counter/v1.py @@ -14,9 +14,8 @@ LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputSelector, - WorkflowParameterSelector, - WorkflowVideoMetadataSelector, + VIDEO_METADATA_KIND, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -49,8 +48,8 @@ class LineCounterManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/line_counter@v1"] - metadata: WorkflowVideoMetadataSelector - detections: StepOutputSelector( + metadata: Selector(kind=[VIDEO_METADATA_KIND]) + detections: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -60,11 +59,11 @@ class LineCounterManifest(WorkflowBlockManifest): examples=["$steps.object_detection_model.predictions"], ) - line_segment: Union[list, StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore + line_segment: Union[list, Selector(kind=[LIST_OF_VALUES_KIND]), Selector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore description="Line in the format [[x1, y1], [x2, y2]] consisting of exactly two points. For line [[0, 100], [100, 100]] line will count objects entering from the bottom as IN", examples=[[[0, 50], [500, 50]], "$inputs.zones"], ) - triggering_anchor: Union[str, WorkflowParameterSelector(kind=[STRING_KIND]), Literal[tuple(sv.Position.list())]] = Field( # type: ignore + triggering_anchor: Union[str, Selector(kind=[STRING_KIND]), Literal[tuple(sv.Position.list())]] = Field( # type: ignore description=f"Point from the detection for triggering line crossing.", default="CENTER", examples=["CENTER"], @@ -85,7 +84,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class LineCounterBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/analytics/line_counter/v2.py b/inference/core/workflows/core_steps/analytics/line_counter/v2.py index b9657842c..db0f4ac70 100644 --- a/inference/core/workflows/core_steps/analytics/line_counter/v2.py +++ b/inference/core/workflows/core_steps/analytics/line_counter/v2.py @@ -14,9 +14,8 @@ LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputSelector, + Selector, WorkflowImageSelector, - WorkflowParameterSelector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -26,6 +25,8 @@ OUTPUT_KEY_COUNT_IN: str = "count_in" OUTPUT_KEY_COUNT_OUT: str = "count_out" +OUTPUT_KEY_DETECTIONS_IN: str = "detections_in" +OUTPUT_KEY_DETECTIONS_OUT: str = "detections_out" IN: str = "in" OUT: str = "out" DETECTIONS_IN_OUT_PARAM: str = "in_out" @@ -55,7 +56,7 @@ class LineCounterManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/line_counter@v2"] image: WorkflowImageSelector - detections: StepOutputSelector( + detections: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -65,11 +66,11 @@ class LineCounterManifest(WorkflowBlockManifest): examples=["$steps.object_detection_model.predictions"], ) - line_segment: Union[list, StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore + line_segment: Union[list, Selector(kind=[LIST_OF_VALUES_KIND]), Selector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore description="Line in the format [[x1, y1], [x2, y2]] consisting of exactly two points. For line [[0, 100], [100, 100]] line will count objects entering from the bottom as IN", examples=[[[0, 50], [500, 50]], "$inputs.zones"], ) - triggering_anchor: Union[str, WorkflowParameterSelector(kind=[STRING_KIND]), Literal[tuple(sv.Position.list())]] = Field( # type: ignore + triggering_anchor: Union[str, Selector(kind=[STRING_KIND]), Literal[tuple(sv.Position.list())]] = Field( # type: ignore description=f"Point from the detection for triggering line crossing.", default="CENTER", examples=["CENTER"], @@ -86,11 +87,25 @@ def describe_outputs(cls) -> List[OutputDefinition]: name=OUTPUT_KEY_COUNT_OUT, kind=[INTEGER_KIND], ), + OutputDefinition( + name=OUTPUT_KEY_DETECTIONS_IN, + kind=[ + OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + ], + ), + OutputDefinition( + name=OUTPUT_KEY_DETECTIONS_OUT, + kind=[ + OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + ], + ), ] @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class LineCounterBlockV2(WorkflowBlock): @@ -136,9 +151,13 @@ def run( ) line_zone = self._batch_of_line_zones[metadata.video_identifier] - line_zone.trigger(detections=detections) + mask_in, mask_out = line_zone.trigger(detections=detections) + detections_in = detections[mask_in] + detections_out = detections[mask_out] return { OUTPUT_KEY_COUNT_IN: line_zone.in_count, OUTPUT_KEY_COUNT_OUT: line_zone.out_count, + OUTPUT_KEY_DETECTIONS_IN: detections_in, + OUTPUT_KEY_DETECTIONS_OUT: detections_out, } diff --git a/inference/core/workflows/core_steps/analytics/path_deviation/v1.py b/inference/core/workflows/core_steps/analytics/path_deviation/v1.py index 276e27caf..e1eebdd60 100644 --- a/inference/core/workflows/core_steps/analytics/path_deviation/v1.py +++ b/inference/core/workflows/core_steps/analytics/path_deviation/v1.py @@ -17,9 +17,8 @@ LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputSelector, - WorkflowParameterSelector, - WorkflowVideoMetadataSelector, + VIDEO_METADATA_KIND, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -52,8 +51,8 @@ class PathDeviationManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/path_deviation_analytics@v1"] - metadata: WorkflowVideoMetadataSelector - detections: StepOutputSelector( + metadata: Selector(kind=[VIDEO_METADATA_KIND]) + detections: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -62,12 +61,12 @@ class PathDeviationManifest(WorkflowBlockManifest): description="Predictions", examples=["$steps.object_detection_model.predictions"], ) - triggering_anchor: Union[str, WorkflowParameterSelector(kind=[STRING_KIND]), Literal[tuple(sv.Position.list())]] = Field( # type: ignore + triggering_anchor: Union[str, Selector(kind=[STRING_KIND]), Literal[tuple(sv.Position.list())]] = Field( # type: ignore description=f"Triggering anchor. Allowed values: {', '.join(sv.Position.list())}", default="CENTER", examples=["CENTER"], ) - reference_path: Union[list, StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore + reference_path: Union[list, Selector(kind=[LIST_OF_VALUES_KIND]), Selector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore description="Reference path in a format [(x1, y1), (x2, y2), (x3, y3), ...]", examples=["$inputs.expected_path"], ) @@ -86,7 +85,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class PathDeviationAnalyticsBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/analytics/path_deviation/v2.py b/inference/core/workflows/core_steps/analytics/path_deviation/v2.py index 4fedef16e..7b12ae2da 100644 --- a/inference/core/workflows/core_steps/analytics/path_deviation/v2.py +++ b/inference/core/workflows/core_steps/analytics/path_deviation/v2.py @@ -17,9 +17,8 @@ LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputSelector, + Selector, WorkflowImageSelector, - WorkflowParameterSelector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -54,7 +53,7 @@ class PathDeviationManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/path_deviation_analytics@v2"] image: WorkflowImageSelector - detections: StepOutputSelector( + detections: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -63,12 +62,12 @@ class PathDeviationManifest(WorkflowBlockManifest): description="Predictions", examples=["$steps.object_detection_model.predictions"], ) - triggering_anchor: Union[str, WorkflowParameterSelector(kind=[STRING_KIND]), Literal[tuple(sv.Position.list())]] = Field( # type: ignore + triggering_anchor: Union[str, Selector(kind=[STRING_KIND]), Literal[tuple(sv.Position.list())]] = Field( # type: ignore description=f"Triggering anchor. Allowed values: {', '.join(sv.Position.list())}", default="CENTER", examples=["CENTER"], ) - reference_path: Union[list, StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore + reference_path: Union[list, Selector(kind=[LIST_OF_VALUES_KIND]), Selector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore description="Reference path in a format [(x1, y1), (x2, y2), (x3, y3), ...]", examples=["$inputs.expected_path"], ) diff --git a/inference/core/workflows/core_steps/analytics/time_in_zone/v1.py b/inference/core/workflows/core_steps/analytics/time_in_zone/v1.py index 5c3c78d4a..295c3d0aa 100644 --- a/inference/core/workflows/core_steps/analytics/time_in_zone/v1.py +++ b/inference/core/workflows/core_steps/analytics/time_in_zone/v1.py @@ -15,15 +15,13 @@ ) from inference.core.workflows.execution_engine.entities.types import ( BOOLEAN_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, - WorkflowVideoMetadataSelector, + VIDEO_METADATA_KIND, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -52,13 +50,13 @@ class TimeInZoneManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/time_in_zone@v1"] - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], ) - metadata: WorkflowVideoMetadataSelector - detections: StepOutputSelector( + metadata: Selector(kind=[VIDEO_METADATA_KIND]) + detections: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -67,21 +65,21 @@ class TimeInZoneManifest(WorkflowBlockManifest): description="Predictions", examples=["$steps.object_detection_model.predictions"], ) - zone: Union[list, StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore + zone: Union[list, Selector(kind=[LIST_OF_VALUES_KIND]), Selector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore description="Zones (one for each batch) in a format [(x1, y1), (x2, y2), (x3, y3), ...]", examples=["$inputs.zones"], ) - triggering_anchor: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + triggering_anchor: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description=f"Triggering anchor. Allowed values: {', '.join(sv.Position.list())}", default="CENTER", examples=["CENTER"], ) - remove_out_of_zone_detections: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( # type: ignore + remove_out_of_zone_detections: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( # type: ignore description=f"If true, detections found outside of zone will be filtered out", default=True, examples=[True, False], ) - reset_out_of_zone_detections: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( # type: ignore + reset_out_of_zone_detections: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( # type: ignore description=f"If true, detections found outside of zone will have time reset", default=True, examples=[True, False], @@ -101,7 +99,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class TimeInZoneBlockV1(WorkflowBlock): @@ -148,7 +146,6 @@ def run( ) self._batch_of_polygon_zones[metadata.video_identifier] = sv.PolygonZone( polygon=np.array(zone), - frame_resolution_wh=image.numpy_image.shape[:-1], triggering_anchors=(sv.Position(triggering_anchor),), ) polygon_zone = self._batch_of_polygon_zones[metadata.video_identifier] diff --git a/inference/core/workflows/core_steps/analytics/time_in_zone/v2.py b/inference/core/workflows/core_steps/analytics/time_in_zone/v2.py index 359f51473..11650c4fa 100644 --- a/inference/core/workflows/core_steps/analytics/time_in_zone/v2.py +++ b/inference/core/workflows/core_steps/analytics/time_in_zone/v2.py @@ -18,9 +18,8 @@ LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputSelector, + Selector, WorkflowImageSelector, - WorkflowParameterSelector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -59,7 +58,7 @@ class TimeInZoneManifest(WorkflowBlockManifest): description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], ) - detections: StepOutputSelector( + detections: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -68,21 +67,21 @@ class TimeInZoneManifest(WorkflowBlockManifest): description="Predictions", examples=["$steps.object_detection_model.predictions"], ) - zone: Union[list, StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore + zone: Union[list, Selector(kind=[LIST_OF_VALUES_KIND]), Selector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore description="Zones (one for each batch) in a format [(x1, y1), (x2, y2), (x3, y3), ...]", examples=["$inputs.zones"], ) - triggering_anchor: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + triggering_anchor: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description=f"Triggering anchor. Allowed values: {', '.join(sv.Position.list())}", default="CENTER", examples=["CENTER"], ) - remove_out_of_zone_detections: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( # type: ignore + remove_out_of_zone_detections: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( # type: ignore description=f"If true, detections found outside of zone will be filtered out", default=True, examples=[True, False], ) - reset_out_of_zone_detections: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( # type: ignore + reset_out_of_zone_detections: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( # type: ignore description=f"If true, detections found outside of zone will have time reset", default=True, examples=[True, False], @@ -102,7 +101,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class TimeInZoneBlockV2(WorkflowBlock): @@ -149,7 +148,6 @@ def run( ) self._batch_of_polygon_zones[metadata.video_identifier] = sv.PolygonZone( polygon=np.array(zone), - frame_resolution_wh=image.numpy_image.shape[:-1], triggering_anchors=(sv.Position(triggering_anchor),), ) polygon_zone = self._batch_of_polygon_zones[metadata.video_identifier] diff --git a/inference/core/workflows/core_steps/classical_cv/camera_focus/v1.py b/inference/core/workflows/core_steps/classical_cv/camera_focus/v1.py index 2c8b71171..3f5ca594a 100644 --- a/inference/core/workflows/core_steps/classical_cv/camera_focus/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/camera_focus/v1.py @@ -14,8 +14,7 @@ from inference.core.workflows.execution_engine.entities.types import ( FLOAT_KIND, IMAGE_KIND, - StepOutputImageSelector, - WorkflowImageSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -45,7 +44,7 @@ class CameraFocusManifest(WorkflowBlockManifest): } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], @@ -69,7 +68,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class CameraFocusBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/classical_cv/contours/v1.py b/inference/core/workflows/core_steps/classical_cv/contours/v1.py index 317cfbdb4..8396bc614 100644 --- a/inference/core/workflows/core_steps/classical_cv/contours/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/contours/v1.py @@ -16,9 +16,7 @@ IMAGE_KIND, INTEGER_KIND, NUMPY_ARRAY_KIND, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -45,14 +43,14 @@ class ImageContoursDetectionManifest(WorkflowBlockManifest): } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], validation_alias=AliasChoices("image", "images"), ) - line_thickness: Union[WorkflowParameterSelector(kind=[INTEGER_KIND]), int] = Field( + line_thickness: Union[Selector(kind=[INTEGER_KIND]), int] = Field( description="Line thickness for drawing contours.", default=3, examples=[3, "$inputs.line_thickness"], @@ -89,7 +87,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ImageContoursDetectionBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/classical_cv/convert_grayscale/v1.py b/inference/core/workflows/core_steps/classical_cv/convert_grayscale/v1.py index 293a15967..1e99f363d 100644 --- a/inference/core/workflows/core_steps/classical_cv/convert_grayscale/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/convert_grayscale/v1.py @@ -12,8 +12,7 @@ ) from inference.core.workflows.execution_engine.entities.types import ( IMAGE_KIND, - StepOutputImageSelector, - WorkflowImageSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -40,7 +39,7 @@ class ConvertGrayscaleManifest(WorkflowBlockManifest): } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], @@ -60,7 +59,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ConvertGrayscaleBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/classical_cv/distance_measurement/v1.py b/inference/core/workflows/core_steps/classical_cv/distance_measurement/v1.py index b51d91c93..0e45909a8 100644 --- a/inference/core/workflows/core_steps/classical_cv/distance_measurement/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/distance_measurement/v1.py @@ -10,8 +10,7 @@ INTEGER_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -46,7 +45,7 @@ class BlockManifest(WorkflowBlockManifest): type: Literal["roboflow_core/distance_measurement@v1"] - predictions: StepOutputSelector( + predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -80,9 +79,7 @@ class BlockManifest(WorkflowBlockManifest): description="Select how to calibrate the measurement of distance between objects.", ) - reference_object_class_name: Union[ - str, WorkflowParameterSelector(kind=[STRING_KIND]) - ] = Field( + reference_object_class_name: Union[str, Selector(kind=[STRING_KIND])] = Field( title="Reference Object Class Name", description="The class name of the reference object.", default="reference-object", @@ -97,7 +94,7 @@ class BlockManifest(WorkflowBlockManifest): }, ) - reference_width: Union[float, WorkflowParameterSelector(kind=[FLOAT_KIND])] = Field( + reference_width: Union[float, Selector(kind=[FLOAT_KIND])] = Field( title="Width", default=2.5, description="Width of the reference object in centimeters", @@ -113,7 +110,7 @@ class BlockManifest(WorkflowBlockManifest): }, ) - reference_height: Union[float, WorkflowParameterSelector(kind=[FLOAT_KIND])] = Field( # type: ignore + reference_height: Union[float, Selector(kind=[FLOAT_KIND])] = Field( # type: ignore title="Height", default=2.5, description="Height of the reference object in centimeters", @@ -129,7 +126,7 @@ class BlockManifest(WorkflowBlockManifest): }, ) - pixel_ratio: Union[float, WorkflowParameterSelector(kind=[FLOAT_KIND])] = Field( + pixel_ratio: Union[float, Selector(kind=[FLOAT_KIND])] = Field( title="Reference Pixel-to-Centimeter Ratio", description="The pixel-to-centimeter ratio of the input image, i.e. 1 centimeter = 100 pixels.", default=100, diff --git a/inference/core/workflows/core_steps/classical_cv/dominant_color/v1.py b/inference/core/workflows/core_steps/classical_cv/dominant_color/v1.py index eeaedd909..205670ab3 100644 --- a/inference/core/workflows/core_steps/classical_cv/dominant_color/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/dominant_color/v1.py @@ -8,11 +8,10 @@ WorkflowImageData, ) from inference.core.workflows.execution_engine.entities.types import ( + IMAGE_KIND, INTEGER_KIND, RGB_COLOR_KIND, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -47,13 +46,13 @@ class DominantColorManifest(WorkflowBlockManifest): "block_type": "classical_computer_vision", } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], validation_alias=AliasChoices("image", "images"), ) - color_clusters: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + color_clusters: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore title="Color Clusters", description="Number of dominant colors to identify. Higher values increase precision but may slow processing.", default=4, @@ -61,7 +60,7 @@ class DominantColorManifest(WorkflowBlockManifest): gt=0, le=10, ) - max_iterations: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + max_iterations: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore title="Max Iterations", description="Max number of iterations to perform. Higher values increase precision but may slow processing.", default=100, @@ -69,7 +68,7 @@ class DominantColorManifest(WorkflowBlockManifest): gt=0, le=500, ) - target_size: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + target_size: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore title="Target Size", description="Sets target for the smallest dimension of the downsampled image in pixels. Lower values increase speed but may reduce precision.", default=100, @@ -86,7 +85,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DominantColorBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py b/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py index d055c7feb..1ab40828a 100644 --- a/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py @@ -15,9 +15,7 @@ IMAGE_KIND, INTEGER_KIND, STRING_KIND, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -45,7 +43,7 @@ class ImageBlurManifest(WorkflowBlockManifest): } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], @@ -53,7 +51,7 @@ class ImageBlurManifest(WorkflowBlockManifest): ) blur_type: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), Literal["average", "gaussian", "median", "bilateral"], ] = Field( default="gaussian", @@ -61,7 +59,7 @@ class ImageBlurManifest(WorkflowBlockManifest): examples=["average", "$inputs.blur_type"], ) - kernel_size: Union[WorkflowParameterSelector(kind=[INTEGER_KIND]), int] = Field( + kernel_size: Union[Selector(kind=[INTEGER_KIND]), int] = Field( default=5, description="Size of the average pooling kernel used for blurring.", examples=[5, "$inputs.kernel_size"], @@ -80,7 +78,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ImageBlurBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/classical_cv/image_preprocessing/v1.py b/inference/core/workflows/core_steps/classical_cv/image_preprocessing/v1.py index dcecd4f5d..d4b4caf59 100644 --- a/inference/core/workflows/core_steps/classical_cv/image_preprocessing/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/image_preprocessing/v1.py @@ -12,9 +12,7 @@ IMAGE_KIND, INTEGER_KIND, STRING_KIND, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -48,7 +46,7 @@ class ImagePreprocessingManifest(WorkflowBlockManifest): }, } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], @@ -57,7 +55,7 @@ class ImagePreprocessingManifest(WorkflowBlockManifest): task_type: Literal["resize", "rotate", "flip"] = Field( description="Preprocessing task to be applied to the image.", ) - width: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + width: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore title="Width", default=640, description="Width of the image to be resized to.", @@ -72,7 +70,7 @@ class ImagePreprocessingManifest(WorkflowBlockManifest): }, }, ) - height: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + height: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore title="Height", default=640, description="Height of the image to be resized to.", @@ -87,7 +85,7 @@ class ImagePreprocessingManifest(WorkflowBlockManifest): }, }, ) - rotation_degrees: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + rotation_degrees: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore title="Degrees of Rotation", description="Positive value to rotate clockwise, negative value to rotate counterclockwise", default=90, @@ -103,7 +101,7 @@ class ImagePreprocessingManifest(WorkflowBlockManifest): } }, ) - flip_type: Union[WorkflowParameterSelector(kind=[STRING_KIND]), Literal["vertical", "horizontal", "both"]] = Field( # type: ignore + flip_type: Union[Selector(kind=[STRING_KIND]), Literal["vertical", "horizontal", "both"]] = Field( # type: ignore title="Flip Type", description="Type of flip to be applied to the image.", default="vertical", @@ -126,7 +124,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ImagePreprocessingBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/classical_cv/pixel_color_count/v1.py b/inference/core/workflows/core_steps/classical_cv/pixel_color_count/v1.py index a6b803608..c7611a9a6 100644 --- a/inference/core/workflows/core_steps/classical_cv/pixel_color_count/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/pixel_color_count/v1.py @@ -9,13 +9,11 @@ WorkflowImageData, ) from inference.core.workflows.execution_engine.entities.types import ( + IMAGE_KIND, INTEGER_KIND, RGB_COLOR_KIND, STRING_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -40,15 +38,15 @@ class ColorPixelCountManifest(WorkflowBlockManifest): "block_type": "classical_computer_vision", } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], validation_alias=AliasChoices("image", "images"), ) target_color: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), - StepOutputSelector(kind=[RGB_COLOR_KIND]), + Selector(kind=[STRING_KIND]), + Selector(kind=[RGB_COLOR_KIND]), str, Tuple[int, int, int], ] = Field( @@ -57,7 +55,7 @@ class ColorPixelCountManifest(WorkflowBlockManifest): "(like (18, 17, 67)).", examples=["#431112", "$inputs.target_color", (18, 17, 67)], ) - tolerance: Union[WorkflowParameterSelector(kind=[INTEGER_KIND]), int] = Field( + tolerance: Union[Selector(kind=[INTEGER_KIND]), int] = Field( default=10, description="Tolerance for color matching.", examples=[10, "$inputs.tolerance"], @@ -65,7 +63,7 @@ class ColorPixelCountManifest(WorkflowBlockManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" @classmethod def describe_outputs(cls) -> List[OutputDefinition]: diff --git a/inference/core/workflows/core_steps/classical_cv/sift/v1.py b/inference/core/workflows/core_steps/classical_cv/sift/v1.py index f0b7c89ca..fdb63df31 100644 --- a/inference/core/workflows/core_steps/classical_cv/sift/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/sift/v1.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional, Type, Union +from typing import List, Literal, Optional, Type import cv2 import numpy as np @@ -15,8 +15,7 @@ IMAGE_KEYPOINTS_KIND, IMAGE_KIND, NUMPY_ARRAY_KIND, - StepOutputImageSelector, - WorkflowImageSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -50,7 +49,7 @@ class SIFTDetectionManifest(WorkflowBlockManifest): } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], @@ -59,7 +58,7 @@ class SIFTDetectionManifest(WorkflowBlockManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" @classmethod def describe_outputs(cls) -> List[OutputDefinition]: diff --git a/inference/core/workflows/core_steps/classical_cv/sift_comparison/v1.py b/inference/core/workflows/core_steps/classical_cv/sift_comparison/v1.py index 890818506..497fa37f6 100644 --- a/inference/core/workflows/core_steps/classical_cv/sift_comparison/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/sift_comparison/v1.py @@ -9,8 +9,7 @@ BOOLEAN_KIND, INTEGER_KIND, NUMPY_ARRAY_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -39,34 +38,30 @@ class SIFTComparisonBlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/sift_comparison@v1"] - descriptor_1: StepOutputSelector(kind=[NUMPY_ARRAY_KIND]) = Field( + descriptor_1: Selector(kind=[NUMPY_ARRAY_KIND]) = Field( description="Reference to SIFT descriptors from the first image to compare", examples=["$steps.sift.descriptors"], ) - descriptor_2: StepOutputSelector(kind=[NUMPY_ARRAY_KIND]) = Field( + descriptor_2: Selector(kind=[NUMPY_ARRAY_KIND]) = Field( description="Reference to SIFT descriptors from the second image to compare", examples=["$steps.sift.descriptors"], ) - good_matches_threshold: Union[ - PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) - ] = Field( + good_matches_threshold: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( default=50, description="Threshold for the number of good matches to consider the images as matching", examples=[50, "$inputs.good_matches_threshold"], ) - ratio_threshold: Union[float, WorkflowParameterSelector(kind=[INTEGER_KIND])] = ( - Field( - default=0.7, - description="Ratio threshold for the ratio test, which is used to filter out poor matches by comparing " - "the distance of the closest match to the distance of the second closest match. A lower " - "ratio indicates stricter filtering.", - examples=[0.7, "$inputs.ratio_threshold"], - ) + ratio_threshold: Union[float, Selector(kind=[INTEGER_KIND])] = Field( + default=0.7, + description="Ratio threshold for the ratio test, which is used to filter out poor matches by comparing " + "the distance of the closest match to the distance of the second closest match. A lower " + "ratio indicates stricter filtering.", + examples=[0.7, "$inputs.ratio_threshold"], ) @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" @classmethod def describe_outputs(cls) -> List[OutputDefinition]: diff --git a/inference/core/workflows/core_steps/classical_cv/sift_comparison/v2.py b/inference/core/workflows/core_steps/classical_cv/sift_comparison/v2.py index 3b401a073..bed195418 100644 --- a/inference/core/workflows/core_steps/classical_cv/sift_comparison/v2.py +++ b/inference/core/workflows/core_steps/classical_cv/sift_comparison/v2.py @@ -17,10 +17,7 @@ INTEGER_KIND, NUMPY_ARRAY_KIND, STRING_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -49,32 +46,20 @@ class SIFTComparisonBlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/sift_comparison@v2"] - input_1: Union[ - WorkflowImageSelector, - StepOutputImageSelector, - StepOutputSelector(kind=[NUMPY_ARRAY_KIND]), - ] = Field( + input_1: Union[Selector(kind=[IMAGE_KIND, NUMPY_ARRAY_KIND]),] = Field( description="Reference to Image or SIFT descriptors from the first image to compare", examples=["$inputs.image1", "$steps.sift.descriptors"], ) - input_2: Union[ - WorkflowImageSelector, - StepOutputImageSelector, - StepOutputSelector(kind=[NUMPY_ARRAY_KIND]), - ] = Field( + input_2: Union[Selector(kind=[IMAGE_KIND, NUMPY_ARRAY_KIND]),] = Field( description="Reference to Image or SIFT descriptors from the second image to compare", examples=["$inputs.image2", "$steps.sift.descriptors"], ) - good_matches_threshold: Union[ - PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) - ] = Field( + good_matches_threshold: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( default=50, description="Threshold for the number of good matches to consider the images as matching", examples=[50, "$inputs.good_matches_threshold"], ) - ratio_threshold: Union[ - float, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]) - ] = Field( + ratio_threshold: Union[float, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( default=0.7, description="Ratio threshold for the ratio test, which is used to filter out poor matches by comparing " "the distance of the closest match to the distance of the second closest match. A lower " @@ -83,13 +68,13 @@ class SIFTComparisonBlockManifest(WorkflowBlockManifest): ) matcher: Union[ Literal["FlannBasedMatcher", "BFMatcher"], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( # type: ignore default="FlannBasedMatcher", description="Matcher to use for comparing the SIFT descriptors", examples=["FlannBasedMatcher", "$inputs.matcher"], ) - visualize: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( + visualize: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=False, description="Whether to visualize the keypoints and matches between the two images", examples=[True, "$inputs.visualize"], @@ -97,7 +82,7 @@ class SIFTComparisonBlockManifest(WorkflowBlockManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" @classmethod def describe_outputs(cls) -> List[OutputDefinition]: diff --git a/inference/core/workflows/core_steps/classical_cv/size_measurement/v1.py b/inference/core/workflows/core_steps/classical_cv/size_measurement/v1.py index 7ee90ea86..522b79a59 100644 --- a/inference/core/workflows/core_steps/classical_cv/size_measurement/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/size_measurement/v1.py @@ -14,8 +14,7 @@ LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -55,7 +54,7 @@ class SizeMeasurementManifest(WorkflowBlockManifest): } ) type: Literal[f"roboflow_core/size_measurement@v1"] - reference_predictions: StepOutputSelector( + reference_predictions: Selector( kind=[ INSTANCE_SEGMENTATION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, @@ -64,7 +63,7 @@ class SizeMeasurementManifest(WorkflowBlockManifest): description="Predictions from the reference object model", examples=["$segmentation.reference_predictions"], ) - object_predictions: StepOutputSelector( + object_predictions: Selector( kind=[ INSTANCE_SEGMENTATION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, @@ -77,7 +76,7 @@ class SizeMeasurementManifest(WorkflowBlockManifest): str, Tuple[float, float], List[float], - WorkflowParameterSelector( + Selector( kind=[STRING_KIND, LIST_OF_VALUES_KIND], ), ] = Field( # type: ignore @@ -93,7 +92,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" def get_detection_dimensions( diff --git a/inference/core/workflows/core_steps/classical_cv/template_matching/v1.py b/inference/core/workflows/core_steps/classical_cv/template_matching/v1.py index 42a1e14d3..de110acdf 100644 --- a/inference/core/workflows/core_steps/classical_cv/template_matching/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/template_matching/v1.py @@ -24,12 +24,11 @@ BOOLEAN_KIND, FLOAT_KIND, FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, INTEGER_KIND, OBJECT_DETECTION_PREDICTION_KIND, FloatZeroToOne, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -66,44 +65,42 @@ class TemplateMatchingManifest(WorkflowBlockManifest): "block_type": "classical_computer_vision", } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], validation_alias=AliasChoices("image", "images"), ) - template: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + template: Selector(kind=[IMAGE_KIND]) = Field( title="Template Image", description="The template image for this step.", examples=["$inputs.template", "$steps.cropping.template"], validation_alias=AliasChoices("template", "templates"), ) - matching_threshold: Union[WorkflowParameterSelector(kind=[FLOAT_KIND]), float] = ( - Field( - title="Matching Threshold", - description="The threshold value for template matching.", - default=0.8, - examples=[0.8, "$inputs.threshold"], - ) + matching_threshold: Union[Selector(kind=[FLOAT_KIND]), float] = Field( + title="Matching Threshold", + description="The threshold value for template matching.", + default=0.8, + examples=[0.8, "$inputs.threshold"], ) - apply_nms: Union[WorkflowParameterSelector(kind=[BOOLEAN_KIND]), bool] = Field( + apply_nms: Union[Selector(kind=[BOOLEAN_KIND]), bool] = Field( title="Apply NMS", description="Flag to decide if NMS should be applied at the output detections.", default=True, examples=["$inputs.apply_nms", False], ) - nms_threshold: Union[ - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), FloatZeroToOne - ] = Field( - title="NMS threshold", - description="The threshold value NMS procedure (if to be applied).", - default=0.5, - examples=["$inputs.nms_threshold", 0.3], + nms_threshold: Union[Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), FloatZeroToOne] = ( + Field( + title="NMS threshold", + description="The threshold value NMS procedure (if to be applied).", + default=0.5, + examples=["$inputs.nms_threshold", 0.3], + ) ) @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" @classmethod def describe_outputs(cls) -> List[OutputDefinition]: diff --git a/inference/core/workflows/core_steps/classical_cv/threshold/v1.py b/inference/core/workflows/core_steps/classical_cv/threshold/v1.py index c3354c70c..f75371037 100644 --- a/inference/core/workflows/core_steps/classical_cv/threshold/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/threshold/v1.py @@ -15,9 +15,7 @@ IMAGE_KIND, INTEGER_KIND, STRING_KIND, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -44,7 +42,7 @@ class ImageThresholdManifest(WorkflowBlockManifest): } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], @@ -52,7 +50,7 @@ class ImageThresholdManifest(WorkflowBlockManifest): ) threshold_type: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), Literal[ "binary", "binary_inv", @@ -69,12 +67,12 @@ class ImageThresholdManifest(WorkflowBlockManifest): examples=["binary", "$inputs.threshold_type"], ) - thresh_value: Union[WorkflowParameterSelector(kind=[INTEGER_KIND]), int] = Field( + thresh_value: Union[Selector(kind=[INTEGER_KIND]), int] = Field( description="Threshold value.", examples=[127, "$inputs.thresh_value"], ) - max_value: Union[WorkflowParameterSelector(kind=[INTEGER_KIND]), int] = Field( + max_value: Union[Selector(kind=[INTEGER_KIND]), int] = Field( description="Maximum value for thresholding", default=255, examples=[255, "$inputs.max_value"], @@ -93,7 +91,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ImageThresholdBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/common/deserializers.py b/inference/core/workflows/core_steps/common/deserializers.py new file mode 100644 index 000000000..9f6ab1c62 --- /dev/null +++ b/inference/core/workflows/core_steps/common/deserializers.py @@ -0,0 +1,443 @@ +import os +from typing import Any, List, Optional, Tuple, Union +from uuid import uuid4 + +import cv2 +import numpy as np +import pybase64 +import supervision as sv +from pydantic import ValidationError + +from inference.core.utils.image_utils import ( + attempt_loading_image_from_string, + load_image_from_url, +) +from inference.core.workflows.core_steps.common.utils import ( + add_inference_keypoints_to_sv_detections, +) +from inference.core.workflows.errors import RuntimeInputError +from inference.core.workflows.execution_engine.constants import ( + BOUNDING_RECT_ANGLE_KEY_IN_INFERENCE_RESPONSE, + BOUNDING_RECT_ANGLE_KEY_IN_SV_DETECTIONS, + BOUNDING_RECT_HEIGHT_KEY_IN_INFERENCE_RESPONSE, + BOUNDING_RECT_HEIGHT_KEY_IN_SV_DETECTIONS, + BOUNDING_RECT_RECT_KEY_IN_INFERENCE_RESPONSE, + BOUNDING_RECT_RECT_KEY_IN_SV_DETECTIONS, + BOUNDING_RECT_WIDTH_KEY_IN_INFERENCE_RESPONSE, + BOUNDING_RECT_WIDTH_KEY_IN_SV_DETECTIONS, + DETECTED_CODE_KEY, + DETECTION_ID_KEY, + IMAGE_DIMENSIONS_KEY, + KEYPOINTS_KEY_IN_INFERENCE_RESPONSE, + PARENT_ID_KEY, + PATH_DEVIATION_KEY_IN_INFERENCE_RESPONSE, + PATH_DEVIATION_KEY_IN_SV_DETECTIONS, + TIME_IN_ZONE_KEY_IN_INFERENCE_RESPONSE, + TIME_IN_ZONE_KEY_IN_SV_DETECTIONS, +) +from inference.core.workflows.execution_engine.entities.base import ( + ImageParentMetadata, + VideoMetadata, + WorkflowImageData, +) + +AnyNumber = Union[int, float] + + +def deserialize_image_kind( + parameter: str, + image: Any, + prevent_local_images_loading: bool = False, +) -> WorkflowImageData: + if isinstance(image, WorkflowImageData): + return image + video_metadata = None + if isinstance(image, dict) and "video_metadata" in image: + video_metadata = deserialize_video_metadata_kind( + parameter=parameter, video_metadata=image["video_metadata"] + ) + if isinstance(image, dict) and isinstance(image.get("value"), np.ndarray): + image = image["value"] + if isinstance(image, np.ndarray): + parent_metadata = ImageParentMetadata(parent_id=parameter) + return WorkflowImageData( + parent_metadata=parent_metadata, + numpy_image=image, + video_metadata=video_metadata, + ) + try: + if isinstance(image, dict): + image = image["value"] + if isinstance(image, str): + base64_image = None + image_reference = None + if image.startswith("http://") or image.startswith("https://"): + image_reference = image + image = load_image_from_url(value=image) + elif not prevent_local_images_loading and os.path.exists(image): + # prevent_local_images_loading is introduced to eliminate + # server vulnerability - namely it prevents local server + # file system from being exploited. + image_reference = image + image = cv2.imread(image) + else: + base64_image = image + image = attempt_loading_image_from_string(image)[0] + parent_metadata = ImageParentMetadata(parent_id=parameter) + return WorkflowImageData( + parent_metadata=parent_metadata, + numpy_image=image, + base64_image=base64_image, + image_reference=image_reference, + video_metadata=video_metadata, + ) + except Exception as error: + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` defined as `WorkflowImage` " + f"that is invalid. Failed on input validation. Details: {error}", + context="workflow_execution | runtime_input_validation", + ) from error + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` defined as `WorkflowImage` " + f"with type {type(image)} that is invalid. Workflows accept only np.arrays, `WorkflowImageData` " + f"and dicts with keys `type` and `value` compatible with `inference` (or list of them).", + context="workflow_execution | runtime_input_validation", + ) + + +def deserialize_video_metadata_kind( + parameter: str, + video_metadata: Any, +) -> VideoMetadata: + if isinstance(video_metadata, VideoMetadata): + return video_metadata + if not isinstance(video_metadata, dict): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` holding " + f"`WorkflowVideoMetadata`, but provided value is not a dict.", + context="workflow_execution | runtime_input_validation", + ) + try: + return VideoMetadata.model_validate(video_metadata) + except ValidationError as error: + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` holding " + f"`WorkflowVideoMetadata`, but provided value is malformed. " + f"See details in inner error.", + context="workflow_execution | runtime_input_validation", + inner_error=error, + ) + + +def deserialize_detections_kind( + parameter: str, + detections: Any, +) -> sv.Detections: + if isinstance(detections, sv.Detections): + return detections + if not isinstance(detections, dict): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"detections, but invalid type of data found.", + context="workflow_execution | runtime_input_validation", + ) + if "predictions" not in detections or "image" not in detections: + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"detections, but dictionary misses required keys.", + context="workflow_execution | runtime_input_validation", + ) + parsed_detections = sv.Detections.from_inference(detections) + if len(parsed_detections) == 0: + return parsed_detections + height, width = detections["image"]["height"], detections["image"]["width"] + image_metadata = np.array([[height, width]] * len(parsed_detections)) + parsed_detections.data[IMAGE_DIMENSIONS_KEY] = image_metadata + detection_ids = [ + detection.get(DETECTION_ID_KEY, str(uuid4())) + for detection in detections["predictions"] + ] + parsed_detections.data[DETECTION_ID_KEY] = np.array(detection_ids) + parent_ids = [ + detection.get(PARENT_ID_KEY, parameter) + for detection in detections["predictions"] + ] + parsed_detections[PARENT_ID_KEY] = np.array(parent_ids) + optional_elements_keys = [ + (PATH_DEVIATION_KEY_IN_INFERENCE_RESPONSE, PATH_DEVIATION_KEY_IN_SV_DETECTIONS), + (TIME_IN_ZONE_KEY_IN_INFERENCE_RESPONSE, TIME_IN_ZONE_KEY_IN_SV_DETECTIONS), + ( + BOUNDING_RECT_ANGLE_KEY_IN_INFERENCE_RESPONSE, + BOUNDING_RECT_ANGLE_KEY_IN_SV_DETECTIONS, + ), + ( + BOUNDING_RECT_RECT_KEY_IN_INFERENCE_RESPONSE, + BOUNDING_RECT_RECT_KEY_IN_SV_DETECTIONS, + ), + ( + BOUNDING_RECT_HEIGHT_KEY_IN_INFERENCE_RESPONSE, + BOUNDING_RECT_HEIGHT_KEY_IN_SV_DETECTIONS, + ), + ( + BOUNDING_RECT_WIDTH_KEY_IN_INFERENCE_RESPONSE, + BOUNDING_RECT_WIDTH_KEY_IN_SV_DETECTIONS, + ), + (DETECTED_CODE_KEY, DETECTED_CODE_KEY), + ] + for raw_detection_key, parsed_detection_key in optional_elements_keys: + parsed_detections = _attach_optional_detection_element( + raw_detections=detections["predictions"], + parsed_detections=parsed_detections, + raw_detection_key=raw_detection_key, + parsed_detection_key=parsed_detection_key, + ) + return _attach_optional_key_points_detections( + raw_detections=detections["predictions"], + parsed_detections=parsed_detections, + ) + + +def _attach_optional_detection_element( + raw_detections: List[dict], + parsed_detections: sv.Detections, + raw_detection_key: str, + parsed_detection_key: str, +) -> sv.Detections: + if raw_detection_key not in raw_detections[0]: + return parsed_detections + result = [] + for detection in raw_detections: + result.append(detection[raw_detection_key]) + parsed_detections.data[parsed_detection_key] = np.array(result) + return parsed_detections + + +def _attach_optional_key_points_detections( + raw_detections: List[dict], + parsed_detections: sv.Detections, +) -> sv.Detections: + if KEYPOINTS_KEY_IN_INFERENCE_RESPONSE not in raw_detections[0]: + return parsed_detections + return add_inference_keypoints_to_sv_detections( + inference_prediction=raw_detections, + detections=parsed_detections, + ) + + +def deserialize_numpy_array(parameter: str, raw_array: Any) -> np.ndarray: + if isinstance(raw_array, np.ndarray): + return raw_array + if not isinstance(raw_array, list): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"numpy array value, but invalid type of data found (`{type(raw_array).__name__}`).", + context="workflow_execution | runtime_input_validation", + ) + return np.array(raw_array) + + +def deserialize_optional_string_kind(parameter: str, value: Any) -> Optional[str]: + if value is None: + return None + return deserialize_string_kind(parameter=parameter, value=value) + + +def deserialize_string_kind(parameter: str, value: Any) -> str: + if not isinstance(value, str): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"string value, but invalid type of data found (`{type(value).__name__}`).", + context="workflow_execution | runtime_input_validation", + ) + return value + + +def deserialize_float_zero_to_one_kind(parameter: str, value: Any) -> float: + value = deserialize_float_kind(parameter=parameter, value=value) + if not (0.0 <= value <= 1.0): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"float value in range [0.0, 1.0], but value out of range detected.", + context="workflow_execution | runtime_input_validation", + ) + return value + + +def deserialize_float_kind(parameter: str, value: Any) -> float: + if not isinstance(value, float) and not isinstance(value, int): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"float value, but invalid type of data found (`{type(value).__name__}`).", + context="workflow_execution | runtime_input_validation", + ) + return float(value) + + +def deserialize_list_of_values_kind(parameter: str, value: Any) -> list: + if not isinstance(value, list) and not isinstance(value, tuple): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"list, but invalid type of data found (`{type(value).__name__}`).", + context="workflow_execution | runtime_input_validation", + ) + if not isinstance(value, list): + return list(value) + return value + + +def deserialize_boolean_kind(parameter: str, value: Any) -> bool: + if not isinstance(value, bool): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"boolean value, but invalid type of data found (`{type(value).__name__}`).", + context="workflow_execution | runtime_input_validation", + ) + return value + + +def deserialize_integer_kind(parameter: str, value: Any) -> int: + if not isinstance(value, int): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"integer value, but invalid type of data found (`{type(value).__name__}`).", + context="workflow_execution | runtime_input_validation", + ) + return value + + +REQUIRED_CLASSIFICATION_PREDICTION_KEYS = { + "image", + "predictions", +} + + +def deserialize_classification_prediction_kind(parameter: str, value: Any) -> dict: + value = deserialize_dictionary_kind(parameter=parameter, value=value) + if any(k not in value for k in REQUIRED_CLASSIFICATION_PREDICTION_KEYS): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"classification prediction value, but found that one of required keys " + f"({list(REQUIRED_CLASSIFICATION_PREDICTION_KEYS)}) " + f"is missing.", + context="workflow_execution | runtime_input_validation", + ) + if "predicted_classes" not in value and ( + "top" not in value or "confidence" not in value + ): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"classification prediction value, but found that passed value misses " + f"prediction details.", + context="workflow_execution | runtime_input_validation", + ) + if "prediction_type" not in value: + value["prediction_type"] = "classification" + if "inference_id" not in value: + value["inference_id"] = str(uuid4()) + if "parent_id" not in value: + value["parent_id"] = parameter + if "root_parent_id" not in value: + value["root_parent_id"] = parameter + return value + + +def deserialize_dictionary_kind(parameter: str, value: Any) -> dict: + if not isinstance(value, dict): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"dict value, but invalid type of data found (`{type(value).__name__}`).", + context="workflow_execution | runtime_input_validation", + ) + return value + + +def deserialize_point_kind(parameter: str, value: Any) -> Tuple[AnyNumber, AnyNumber]: + if not isinstance(value, list) and not isinstance(value, tuple): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"point coordinates, but invalid type of data found (`{type(value).__name__}`).", + context="workflow_execution | runtime_input_validation", + ) + if len(value) < 2: + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"point coordinates, but missing point coordinates detected.", + context="workflow_execution | runtime_input_validation", + ) + value = tuple(value[:2]) + if any(not _is_number(e) for e in value): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"point coordinates, but at least one of the coordinate is not number", + context="workflow_execution | runtime_input_validation", + ) + return value + + +def deserialize_zone_kind( + parameter: str, value: Any +) -> List[List[Tuple[AnyNumber, AnyNumber]]]: + if not isinstance(value, list) or len(value) < 3: + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"zone coordinates, but defined zone is not a list with at least 3 points coordinates.", + context="workflow_execution | runtime_input_validation", + ) + if any( + (not isinstance(e, list) and not isinstance(e, tuple)) or len(e) != 2 + for e in value + ): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"zone coordinates, but defined zone contains at least one element which is not a point with" + f"exactly two coordinates (x, y).", + context="workflow_execution | runtime_input_validation", + ) + if any(not _is_number(e[0]) or not _is_number(e[1]) for e in value): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"zone coordinates, but defined zone contains at least one element which is not a point with" + f"exactly two coordinates (x, y) being numbers.", + context="workflow_execution | runtime_input_validation", + ) + return value + + +def deserialize_rgb_color_kind( + parameter: str, value: Any +) -> Union[Tuple[int, int, int], str]: + if ( + not isinstance(value, list) + and not isinstance(value, tuple) + and not isinstance(value, str) + ): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"RGB color, but invalid type of data found (`{type(value).__name__}`).", + context="workflow_execution | runtime_input_validation", + ) + if isinstance(value, str): + return value + if len(value) < 3: + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"RGB color, but not all colors defined.", + context="workflow_execution | runtime_input_validation", + ) + return tuple(value[:3]) + + +def deserialize_bytes_kind(parameter: str, value: Any) -> bytes: + if not isinstance(value, str) and not isinstance(value, bytes): + raise RuntimeInputError( + public_message=f"Detected runtime parameter `{parameter}` declared to hold " + f"bytes string, but invalid type of data found (`{type(value).__name__}`).", + context="workflow_execution | runtime_input_validation", + ) + if isinstance(value, bytes): + return value + return pybase64.b64decode(value) + + +def _is_number(value: Any) -> bool: + return isinstance(value, int) or isinstance(value, float) diff --git a/inference/core/workflows/core_steps/common/serializers.py b/inference/core/workflows/core_steps/common/serializers.py index 736261d00..aa0cfea6f 100644 --- a/inference/core/workflows/core_steps/common/serializers.py +++ b/inference/core/workflows/core_steps/common/serializers.py @@ -1,4 +1,4 @@ -from typing import Any, Dict +from typing import Any, Dict, List import numpy as np import supervision as sv @@ -35,7 +35,10 @@ X_KEY, Y_KEY, ) -from inference.core.workflows.execution_engine.entities.base import WorkflowImageData +from inference.core.workflows.execution_engine.entities.base import ( + VideoMetadata, + WorkflowImageData, +) def serialise_sv_detections(detections: sv.Detections) -> dict: @@ -143,4 +146,37 @@ def serialise_image(image: WorkflowImageData) -> Dict[str, Any]: return { "type": "base64", "value": image.base64_image, + "video_metadata": image.video_metadata.dict(), } + + +def serialize_video_metadata_kind(video_metadata: VideoMetadata) -> dict: + return video_metadata.dict() + + +def serialize_wildcard_kind(value: Any) -> Any: + if isinstance(value, WorkflowImageData): + value = serialise_image(image=value) + elif isinstance(value, dict): + value = serialize_dict(elements=value) + elif isinstance(value, list): + value = serialize_list(elements=value) + elif isinstance(value, sv.Detections): + value = serialise_sv_detections(detections=value) + return value + + +def serialize_list(elements: List[Any]) -> List[Any]: + result = [] + for element in elements: + element = serialize_wildcard_kind(value=element) + result.append(element) + return result + + +def serialize_dict(elements: Dict[str, Any]) -> Dict[str, Any]: + serialized_result = {} + for key, value in elements.items(): + value = serialize_wildcard_kind(value=value) + serialized_result[key] = value + return serialized_result diff --git a/inference/core/workflows/core_steps/common/utils.py b/inference/core/workflows/core_steps/common/utils.py index 138afe9c0..d8fc916f4 100644 --- a/inference/core/workflows/core_steps/common/utils.py +++ b/inference/core/workflows/core_steps/common/utils.py @@ -100,7 +100,7 @@ def convert_inference_detections_batch_to_sv_detections( detections = sv.Detections.from_inference(p) parent_ids = [d.get(PARENT_ID_KEY, "") for d in p[predictions_key]] detection_ids = [ - d.get(DETECTION_ID_KEY, str(uuid.uuid4)) for d in p[predictions_key] + d.get(DETECTION_ID_KEY, str(uuid.uuid4())) for d in p[predictions_key] ] detections[DETECTION_ID_KEY] = np.array(detection_ids) detections[PARENT_ID_KEY] = np.array(parent_ids) diff --git a/inference/core/workflows/core_steps/flow_control/continue_if/v1.py b/inference/core/workflows/core_steps/flow_control/continue_if/v1.py index e3a78607b..275cb2a54 100644 --- a/inference/core/workflows/core_steps/flow_control/continue_if/v1.py +++ b/inference/core/workflows/core_steps/flow_control/continue_if/v1.py @@ -10,10 +10,8 @@ ) from inference.core.workflows.execution_engine.entities.base import OutputDefinition from inference.core.workflows.execution_engine.entities.types import ( - StepOutputSelector, + Selector, StepSelector, - WorkflowImageSelector, - WorkflowParameterSelector, ) from inference.core.workflows.execution_engine.v1.entities import FlowControl from inference.core.workflows.prototypes.block import ( @@ -63,7 +61,7 @@ class BlockManifest(WorkflowBlockManifest): ) evaluation_parameters: Dict[ str, - Union[WorkflowImageSelector, WorkflowParameterSelector(), StepOutputSelector()], + Selector(), ] = Field( description="References to additional parameters that may be provided in runtime to parametrise operations", examples=[{"left": "$inputs.some"}], @@ -80,7 +78,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ContinueIfBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/flow_control/rate_limiter/v1.py b/inference/core/workflows/core_steps/flow_control/rate_limiter/v1.py index 17c03e324..df381e5dc 100644 --- a/inference/core/workflows/core_steps/flow_control/rate_limiter/v1.py +++ b/inference/core/workflows/core_steps/flow_control/rate_limiter/v1.py @@ -5,10 +5,8 @@ from inference.core.workflows.execution_engine.entities.base import OutputDefinition from inference.core.workflows.execution_engine.entities.types import ( - StepOutputSelector, + Selector, StepSelector, - WorkflowImageSelector, - WorkflowParameterSelector, ) from inference.core.workflows.execution_engine.v1.entities import FlowControl from inference.core.workflows.prototypes.block import ( @@ -61,9 +59,7 @@ class RateLimiterManifest(WorkflowBlockManifest): default=1.0, ge=0.0, ) - depends_on: Union[ - WorkflowImageSelector, WorkflowParameterSelector(), StepOutputSelector() - ] = Field( + depends_on: Selector() = Field( description="Reference to any output of the the step which immediately preceeds this branch.", examples=["$steps.model"], ) @@ -78,7 +74,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RateLimiterBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/formatters/csv/v1.py b/inference/core/workflows/core_steps/formatters/csv/v1.py index efc1484fd..aa6bff7de 100644 --- a/inference/core/workflows/core_steps/formatters/csv/v1.py +++ b/inference/core/workflows/core_steps/formatters/csv/v1.py @@ -16,12 +16,8 @@ OutputDefinition, ) from inference.core.workflows.execution_engine.entities.types import ( - BOOLEAN_KIND, - INTEGER_KIND, STRING_KIND, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -140,9 +136,7 @@ class BlockManifest(WorkflowBlockManifest): columns_data: Dict[ str, Union[ - WorkflowImageSelector, - WorkflowParameterSelector(), - StepOutputSelector(), + Selector(), str, int, float, @@ -179,8 +173,8 @@ def protect_timestamp_column(cls, value: dict) -> dict: return value @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches_and_scalars(cls) -> List[str]: + return ["columns_data"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -190,7 +184,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class CSVFormatterBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/formatters/expression/v1.py b/inference/core/workflows/core_steps/formatters/expression/v1.py index 7bbc2d7fc..f5658d46b 100644 --- a/inference/core/workflows/core_steps/formatters/expression/v1.py +++ b/inference/core/workflows/core_steps/formatters/expression/v1.py @@ -15,11 +15,7 @@ build_operations_chain, ) from inference.core.workflows.execution_engine.entities.base import OutputDefinition -from inference.core.workflows.execution_engine.entities.types import ( - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, -) +from inference.core.workflows.execution_engine.entities.types import Selector from inference.core.workflows.prototypes.block import ( BlockResult, WorkflowBlock, @@ -109,7 +105,7 @@ class BlockManifest(WorkflowBlockManifest): type: Literal["roboflow_core/expression@v1", "Expression"] data: Dict[ str, - Union[WorkflowImageSelector, WorkflowParameterSelector(), StepOutputSelector()], + Union[Selector()], ] = Field( description="References data to be used to construct results", examples=[ @@ -142,7 +138,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ExpressionBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/formatters/first_non_empty_or_default/v1.py b/inference/core/workflows/core_steps/formatters/first_non_empty_or_default/v1.py index b3c05ce53..558756f7a 100644 --- a/inference/core/workflows/core_steps/formatters/first_non_empty_or_default/v1.py +++ b/inference/core/workflows/core_steps/formatters/first_non_empty_or_default/v1.py @@ -6,7 +6,7 @@ Batch, OutputDefinition, ) -from inference.core.workflows.execution_engine.entities.types import StepOutputSelector +from inference.core.workflows.execution_engine.entities.types import Selector from inference.core.workflows.prototypes.block import ( BlockResult, WorkflowBlock, @@ -35,7 +35,7 @@ class BlockManifest(WorkflowBlockManifest): type: Literal[ "roboflow_core/first_non_empty_or_default@v1", "FirstNonEmptyOrDefault" ] - data: List[StepOutputSelector()] = Field( + data: List[Selector()] = Field( description="Reference data to replace empty values", examples=["$steps.my_step.predictions"], min_items=1, @@ -56,7 +56,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class FirstNonEmptyOrDefaultBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/formatters/json_parser/v1.py b/inference/core/workflows/core_steps/formatters/json_parser/v1.py index 0e1a0f1b3..2907d5d1c 100644 --- a/inference/core/workflows/core_steps/formatters/json_parser/v1.py +++ b/inference/core/workflows/core_steps/formatters/json_parser/v1.py @@ -10,7 +10,7 @@ from inference.core.workflows.execution_engine.entities.types import ( BOOLEAN_KIND, LANGUAGE_MODEL_OUTPUT_KIND, - StepOutputSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -63,7 +63,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/json_parser@v1"] - raw_json: StepOutputSelector(kind=[LANGUAGE_MODEL_OUTPUT_KIND]) = Field( + raw_json: Selector(kind=[LANGUAGE_MODEL_OUTPUT_KIND]) = Field( description="The string with raw JSON to parse.", examples=[["$steps.lmm.output"]], ) @@ -91,7 +91,7 @@ def get_actual_outputs(self) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class JSONParserBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/formatters/property_definition/v1.py b/inference/core/workflows/core_steps/formatters/property_definition/v1.py index 8fb0bc05f..6570b3a39 100644 --- a/inference/core/workflows/core_steps/formatters/property_definition/v1.py +++ b/inference/core/workflows/core_steps/formatters/property_definition/v1.py @@ -9,10 +9,7 @@ build_operations_chain, ) from inference.core.workflows.execution_engine.entities.base import OutputDefinition -from inference.core.workflows.execution_engine.entities.types import ( - StepOutputSelector, - WorkflowImageSelector, -) +from inference.core.workflows.execution_engine.entities.types import Selector from inference.core.workflows.prototypes.block import ( BlockResult, WorkflowBlock, @@ -57,7 +54,7 @@ class BlockManifest(WorkflowBlockManifest): "PropertyDefinition", "PropertyExtraction", ] - data: Union[WorkflowImageSelector, StepOutputSelector()] = Field( + data: Selector() = Field( description="Reference data to extract property from", examples=["$steps.my_step.predictions"], ) @@ -74,7 +71,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class PropertyDefinitionBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/formatters/vlm_as_classifier/v1.py b/inference/core/workflows/core_steps/formatters/vlm_as_classifier/v1.py index ee07cd771..ac6e10c46 100644 --- a/inference/core/workflows/core_steps/formatters/vlm_as_classifier/v1.py +++ b/inference/core/workflows/core_steps/formatters/vlm_as_classifier/v1.py @@ -13,13 +13,11 @@ from inference.core.workflows.execution_engine.entities.types import ( BOOLEAN_KIND, CLASSIFICATION_PREDICTION_KIND, + IMAGE_KIND, LANGUAGE_MODEL_OUTPUT_KIND, LIST_OF_VALUES_KIND, STRING_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -66,18 +64,18 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/vlm_as_classifier@v1"] - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="The image which was the base to generate VLM prediction", examples=["$inputs.image", "$steps.cropping.crops"], ) - vlm_output: StepOutputSelector(kind=[LANGUAGE_MODEL_OUTPUT_KIND]) = Field( + vlm_output: Selector(kind=[LANGUAGE_MODEL_OUTPUT_KIND]) = Field( title="VLM Output", description="The string with raw classification prediction to parse.", examples=[["$steps.lmm.output"]], ) classes: Union[ - WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), - StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), + Selector(kind=[LIST_OF_VALUES_KIND]), + Selector(kind=[LIST_OF_VALUES_KIND]), List[str], ] = Field( description="List of all classes used by the model, required to " @@ -95,7 +93,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class VLMAsClassifierBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/formatters/vlm_as_classifier/v2.py b/inference/core/workflows/core_steps/formatters/vlm_as_classifier/v2.py new file mode 100644 index 000000000..d751ba91a --- /dev/null +++ b/inference/core/workflows/core_steps/formatters/vlm_as_classifier/v2.py @@ -0,0 +1,267 @@ +import json +import logging +import re +from typing import Dict, List, Literal, Optional, Tuple, Type, Union +from uuid import uuid4 + +from pydantic import ConfigDict, Field + +from inference.core.workflows.execution_engine.entities.base import ( + OutputDefinition, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + CLASSIFICATION_PREDICTION_KIND, + IMAGE_KIND, + INFERENCE_ID_KIND, + LANGUAGE_MODEL_OUTPUT_KIND, + LIST_OF_VALUES_KIND, + Selector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) + +JSON_MARKDOWN_BLOCK_PATTERN = re.compile(r"```json([\s\S]*?)```", flags=re.IGNORECASE) + +LONG_DESCRIPTION = """ +The block expects string input that would be produced by blocks exposing Large Language Models (LLMs) and +Visual Language Models (VLMs). Input is parsed to classification prediction and returned as block output. + +Accepted formats: + +- valid JSON strings + +- JSON documents wrapped with Markdown tags (very common for GPT responses) + +Example: +``` +{"my": "json"} +``` + +**Details regarding block behavior:** + +- `error_status` is set `True` whenever parsing cannot be completed + +- in case of multiple markdown blocks with raw JSON content - only first will be parsed +""" + +SHORT_DESCRIPTION = "Parses raw string into classification prediction." + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "VLM as Classifier", + "version": "v2", + "short_description": SHORT_DESCRIPTION, + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "formatter", + } + ) + type: Literal["roboflow_core/vlm_as_classifier@v2"] + image: Selector(kind=[IMAGE_KIND]) = Field( + description="The image which was the base to generate VLM prediction", + examples=["$inputs.image", "$steps.cropping.crops"], + ) + vlm_output: Selector(kind=[LANGUAGE_MODEL_OUTPUT_KIND]) = Field( + title="VLM Output", + description="The string with raw classification prediction to parse.", + examples=[["$steps.lmm.output"]], + ) + classes: Union[ + Selector(kind=[LIST_OF_VALUES_KIND]), + Selector(kind=[LIST_OF_VALUES_KIND]), + List[str], + ] = Field( + description="List of all classes used by the model, required to " + "generate mapping between class name and class id.", + examples=[["$steps.lmm.classes", "$inputs.classes", ["class_a", "class_b"]]], + ) + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="error_status", kind=[BOOLEAN_KIND]), + OutputDefinition(name="predictions", kind=[CLASSIFICATION_PREDICTION_KIND]), + OutputDefinition(name="inference_id", kind=[INFERENCE_ID_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class VLMAsClassifierBlockV2(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run( + self, + image: WorkflowImageData, + vlm_output: str, + classes: List[str], + ) -> BlockResult: + inference_id = f"{uuid4()}" + error_status, parsed_data = string2json( + raw_json=vlm_output, + ) + if error_status: + return { + "error_status": True, + "predictions": None, + "inference_id": inference_id, + } + if "class_name" in parsed_data and "confidence" in parsed_data: + return parse_multi_class_classification_results( + image=image, + results=parsed_data, + classes=classes, + inference_id=inference_id, + ) + if "predicted_classes" in parsed_data: + return parse_multi_label_classification_results( + image=image, + results=parsed_data, + classes=classes, + inference_id=inference_id, + ) + return { + "error_status": True, + "predictions": None, + "inference_id": inference_id, + } + + +def string2json( + raw_json: str, +) -> Tuple[bool, dict]: + json_blocks_found = JSON_MARKDOWN_BLOCK_PATTERN.findall(raw_json) + if len(json_blocks_found) == 0: + return try_parse_json(raw_json) + first_block = json_blocks_found[0] + return try_parse_json(first_block) + + +def try_parse_json(content: str) -> Tuple[bool, dict]: + try: + return False, json.loads(content) + except Exception as error: + logging.warning( + f"Could not parse JSON to dict in `roboflow_core/vlm_as_classifier@v1` block. " + f"Error type: {error.__class__.__name__}. Details: {error}" + ) + return True, {} + + +def parse_multi_class_classification_results( + image: WorkflowImageData, + results: dict, + classes: List[str], + inference_id: str, +) -> dict: + try: + class2id_mapping = create_classes_index(classes=classes) + height, width = image.numpy_image.shape[:2] + top_class = results["class_name"] + confidences = {top_class: scale_confidence(results["confidence"])} + predictions = [] + if top_class not in class2id_mapping: + predictions.append( + { + "class": top_class, + "class_id": -1, + "confidence": confidences.get(top_class, 0.0), + } + ) + for class_name, class_id in class2id_mapping.items(): + predictions.append( + { + "class": class_name, + "class_id": class_id, + "confidence": confidences.get(class_name, 0.0), + } + ) + parsed_prediction = { + "image": {"width": width, "height": height}, + "predictions": predictions, + "top": top_class, + "confidence": confidences[top_class], + "inference_id": inference_id, + "parent_id": image.parent_metadata.parent_id, + } + return { + "error_status": False, + "predictions": parsed_prediction, + "inference_id": inference_id, + } + except Exception as error: + logging.warning( + f"Could not parse multi-class classification results in `roboflow_core/vlm_as_classifier@v1` block. " + f"Error type: {error.__class__.__name__}. Details: {error}" + ) + return {"error_status": True, "predictions": None, "inference_id": inference_id} + + +def parse_multi_label_classification_results( + image: WorkflowImageData, + results: dict, + classes: List[str], + inference_id: str, +) -> dict: + try: + class2id_mapping = create_classes_index(classes=classes) + height, width = image.numpy_image.shape[:2] + predicted_classes_confidences = {} + for prediction in results["predicted_classes"]: + if prediction["class"] not in class2id_mapping: + class2id_mapping[prediction["class"]] = -1 + if prediction["class"] in predicted_classes_confidences: + old_confidence = predicted_classes_confidences[prediction["class"]] + new_confidence = scale_confidence(value=prediction["confidence"]) + predicted_classes_confidences[prediction["class"]] = max( + old_confidence, new_confidence + ) + else: + predicted_classes_confidences[prediction["class"]] = scale_confidence( + value=prediction["confidence"] + ) + predictions = { + class_name: { + "confidence": predicted_classes_confidences.get(class_name, 0.0), + "class_id": class_id, + } + for class_name, class_id in class2id_mapping.items() + } + parsed_prediction = { + "image": {"width": width, "height": height}, + "predictions": predictions, + "predicted_classes": list(predicted_classes_confidences.keys()), + "inference_id": inference_id, + "parent_id": image.parent_metadata.parent_id, + } + return { + "error_status": False, + "predictions": parsed_prediction, + "inference_id": inference_id, + } + except Exception as error: + logging.warning( + f"Could not parse multi-label classification results in `roboflow_core/vlm_as_classifier@v1` block. " + f"Error type: {error.__class__.__name__}. Details: {error}" + ) + return {"error_status": True, "predictions": None, "inference_id": inference_id} + + +def create_classes_index(classes: List[str]) -> Dict[str, int]: + return {class_name: idx for idx, class_name in enumerate(classes)} + + +def scale_confidence(value: float) -> float: + return min(max(float(value), 0.0), 1.0) diff --git a/inference/core/workflows/core_steps/formatters/vlm_as_detector/v1.py b/inference/core/workflows/core_steps/formatters/vlm_as_detector/v1.py index 8ebb1c7af..48c50e822 100644 --- a/inference/core/workflows/core_steps/formatters/vlm_as_detector/v1.py +++ b/inference/core/workflows/core_steps/formatters/vlm_as_detector/v1.py @@ -27,14 +27,12 @@ ) from inference.core.workflows.execution_engine.entities.types import ( BOOLEAN_KIND, + IMAGE_KIND, LANGUAGE_MODEL_OUTPUT_KIND, LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -90,22 +88,23 @@ class BlockManifest(WorkflowBlockManifest): "long_description": LONG_DESCRIPTION, "license": "Apache-2.0", "block_type": "formatter", - } + }, + protected_namespaces=(), ) type: Literal["roboflow_core/vlm_as_detector@v1"] - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="The image which was the base to generate VLM prediction", examples=["$inputs.image", "$steps.cropping.crops"], ) - vlm_output: StepOutputSelector(kind=[LANGUAGE_MODEL_OUTPUT_KIND]) = Field( + vlm_output: Selector(kind=[LANGUAGE_MODEL_OUTPUT_KIND]) = Field( title="VLM Output", description="The string with raw classification prediction to parse.", examples=[["$steps.lmm.output"]], ) classes: Optional[ Union[ - WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), - StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), + Selector(kind=[LIST_OF_VALUES_KIND]), + Selector(kind=[LIST_OF_VALUES_KIND]), List[str], ] ] = Field( @@ -157,7 +156,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class VLMAsDetectorBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/formatters/vlm_as_detector/v2.py b/inference/core/workflows/core_steps/formatters/vlm_as_detector/v2.py new file mode 100644 index 000000000..f752bf8e0 --- /dev/null +++ b/inference/core/workflows/core_steps/formatters/vlm_as_detector/v2.py @@ -0,0 +1,376 @@ +import hashlib +import json +import logging +import re +from functools import partial +from typing import Dict, List, Literal, Optional, Tuple, Type, Union +from uuid import uuid4 + +import numpy as np +import supervision as sv +from pydantic import ConfigDict, Field, model_validator +from supervision.config import CLASS_NAME_DATA_FIELD + +from inference.core.workflows.core_steps.common.utils import ( + attach_parents_coordinates_to_sv_detections, +) +from inference.core.workflows.core_steps.common.vlms import VLM_TASKS_METADATA +from inference.core.workflows.execution_engine.constants import ( + DETECTION_ID_KEY, + IMAGE_DIMENSIONS_KEY, + INFERENCE_ID_KEY, + PREDICTION_TYPE_KEY, +) +from inference.core.workflows.execution_engine.entities.base import ( + OutputDefinition, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + IMAGE_KIND, + INFERENCE_ID_KIND, + LANGUAGE_MODEL_OUTPUT_KIND, + LIST_OF_VALUES_KIND, + OBJECT_DETECTION_PREDICTION_KIND, + Selector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) + +JSON_MARKDOWN_BLOCK_PATTERN = re.compile(r"```json([\s\S]*?)```", flags=re.IGNORECASE) + +LONG_DESCRIPTION = """ +The block expects string input that would be produced by blocks exposing Large Language Models (LLMs) and +Visual Language Models (VLMs). Input is parsed to object-detection prediction and returned as block output. + +Accepted formats: + +- valid JSON strings + +- JSON documents wrapped with Markdown tags + +Example +``` +{"my": "json"} +``` + +**Details regarding block behavior:** + +- `error_status` is set `True` whenever parsing cannot be completed + +- in case of multiple markdown blocks with raw JSON content - only first will be parsed +""" + +SHORT_DESCRIPTION = "Parses raw string into object-detection prediction." + +SUPPORTED_TASKS = { + "object-detection", + "object-detection-and-caption", + "open-vocabulary-object-detection", + "phrase-grounded-object-detection", + "region-proposal", + "ocr-with-text-detection", +} +RELEVANT_TASKS_METADATA = { + k: v for k, v in VLM_TASKS_METADATA.items() if k in SUPPORTED_TASKS +} + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "VLM as Detector", + "version": "v2", + "short_description": SHORT_DESCRIPTION, + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "formatter", + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/vlm_as_detector@v2"] + image: Selector(kind=[IMAGE_KIND]) = Field( + description="The image which was the base to generate VLM prediction", + examples=["$inputs.image", "$steps.cropping.crops"], + ) + vlm_output: Selector(kind=[LANGUAGE_MODEL_OUTPUT_KIND]) = Field( + title="VLM Output", + description="The string with raw classification prediction to parse.", + examples=[["$steps.lmm.output"]], + ) + classes: Optional[ + Union[ + Selector(kind=[LIST_OF_VALUES_KIND]), + Selector(kind=[LIST_OF_VALUES_KIND]), + List[str], + ] + ] = Field( + description="List of all classes used by the model, required to " + "generate mapping between class name and class id.", + examples=[["$steps.lmm.classes", "$inputs.classes", ["class_a", "class_b"]]], + json_schema_extra={ + "relevant_for": { + "model_type": { + "values": ["google-gemini", "anthropic-claude"], + "required": True, + }, + } + }, + ) + model_type: Literal["google-gemini", "anthropic-claude", "florence-2"] = Field( + description="Type of the model that generated prediction", + examples=[["google-gemini", "anthropic-claude", "florence-2"]], + ) + task_type: Literal[tuple(SUPPORTED_TASKS)] = Field( + description="Task type to performed by model.", + json_schema_extra={ + "values_metadata": RELEVANT_TASKS_METADATA, + }, + ) + + @model_validator(mode="after") + def validate(self) -> "BlockManifest": + if (self.model_type, self.task_type) not in REGISTERED_PARSERS: + raise ValueError( + f"Could not parse result of task {self.task_type} for model {self.model_type}" + ) + if self.model_type != "florence-2" and self.classes is None: + raise ValueError( + "Must pass list of classes to this block when using gemini or claude" + ) + + return self + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="error_status", kind=[BOOLEAN_KIND]), + OutputDefinition( + name="predictions", kind=[OBJECT_DETECTION_PREDICTION_KIND] + ), + OutputDefinition(name="inference_id", kind=[INFERENCE_ID_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class VLMAsDetectorBlockV2(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run( + self, + image: WorkflowImageData, + vlm_output: str, + classes: Optional[List[str]], + model_type: str, + task_type: str, + ) -> BlockResult: + inference_id = f"{uuid4()}" + error_status, parsed_data = string2json( + raw_json=vlm_output, + ) + if error_status: + return { + "error_status": True, + "predictions": None, + "inference_id": inference_id, + } + try: + predictions = REGISTERED_PARSERS[(model_type, task_type)]( + image=image, + parsed_data=parsed_data, + classes=classes, + inference_id=inference_id, + ) + return { + "error_status": False, + "predictions": predictions, + "inference_id": inference_id, + } + except Exception as error: + logging.warning( + f"Could not parse VLM prediction for model {model_type} and task {task_type} " + f"in `roboflow_core/vlm_as_detector@v1` block. " + f"Error type: {error.__class__.__name__}. Details: {error}" + ) + return { + "error_status": True, + "predictions": None, + "inference_id": inference_id, + } + + +def string2json( + raw_json: str, +) -> Tuple[bool, dict]: + json_blocks_found = JSON_MARKDOWN_BLOCK_PATTERN.findall(raw_json) + if len(json_blocks_found) == 0: + return try_parse_json(raw_json) + first_block = json_blocks_found[0] + return try_parse_json(first_block) + + +def try_parse_json(content: str) -> Tuple[bool, dict]: + try: + return False, json.loads(content) + except Exception as error: + logging.warning( + f"Could not parse JSON to dict in `roboflow_core/vlm_as_detector@v1` block. " + f"Error type: {error.__class__.__name__}. Details: {error}" + ) + return True, {} + + +def parse_gemini_object_detection_response( + image: WorkflowImageData, + parsed_data: dict, + classes: List[str], + inference_id: str, +) -> sv.Detections: + class_name2id = create_classes_index(classes=classes) + image_height, image_width = image.numpy_image.shape[:2] + if len(parsed_data["detections"]) == 0: + return sv.Detections.empty() + xyxy, class_id, class_name, confidence = [], [], [], [] + for detection in parsed_data["detections"]: + xyxy.append( + [ + detection["x_min"] * image_width, + detection["y_min"] * image_height, + detection["x_max"] * image_width, + detection["y_max"] * image_height, + ] + ) + class_id.append(class_name2id.get(detection["class_name"], -1)) + class_name.append(detection["class_name"]) + confidence.append(scale_confidence(detection.get("confidence", 1.0))) + xyxy = np.array(xyxy).round(0) if len(xyxy) > 0 else np.empty((0, 4)) + confidence = np.array(confidence) if len(confidence) > 0 else np.empty(0) + class_id = np.array(class_id).astype(int) if len(class_id) > 0 else np.empty(0) + class_name = np.array(class_name) if len(class_name) > 0 else np.empty(0) + detection_ids = np.array([str(uuid4()) for _ in range(len(xyxy))]) + dimensions = np.array([[image_height, image_width]] * len(xyxy)) + inference_ids = np.array([inference_id] * len(xyxy)) + prediction_type = np.array(["object-detection"] * len(xyxy)) + data = { + CLASS_NAME_DATA_FIELD: class_name, + IMAGE_DIMENSIONS_KEY: dimensions, + INFERENCE_ID_KEY: inference_ids, + DETECTION_ID_KEY: detection_ids, + PREDICTION_TYPE_KEY: prediction_type, + } + detections = sv.Detections( + xyxy=xyxy, + confidence=confidence, + class_id=class_id, + mask=None, + tracker_id=None, + data=data, + ) + return attach_parents_coordinates_to_sv_detections( + detections=detections, + image=image, + ) + + +def create_classes_index(classes: List[str]) -> Dict[str, int]: + return {class_name: idx for idx, class_name in enumerate(classes)} + + +def scale_confidence(value: float) -> float: + return min(max(float(value), 0.0), 1.0) + + +def parse_florence2_object_detection_response( + image: WorkflowImageData, + parsed_data: dict, + classes: Optional[List[str]], + inference_id: str, + florence_task_type: str, +): + image_height, image_width = image.numpy_image.shape[:2] + detections = sv.Detections.from_lmm( + "florence_2", + result={florence_task_type: parsed_data}, + resolution_wh=(image_width, image_height), + ) + detections.class_id = np.array([0] * len(detections)) + if florence_task_type == "": + detections.data["class_name"] = np.array(["roi"] * len(detections)) + if florence_task_type in {"", ""}: + unique_class_names = set(detections.data.get("class_name", [])) + class_name_to_id = { + name: get_4digit_from_md5(name) for name in unique_class_names + } + class_ids = [ + class_name_to_id.get(name, -1) + for name in detections.data.get("class_name", ["unknown"] * len(detections)) + ] + detections.class_id = np.array(class_ids) + if florence_task_type in "": + class_name_to_id = {name: idx for idx, name in enumerate(classes)} + class_ids = [ + class_name_to_id.get(name, -1) + for name in detections.data.get("class_name", ["unknown"] * len(detections)) + ] + detections.class_id = np.array(class_ids) + dimensions = np.array([[image_height, image_width]] * len(detections)) + detection_ids = np.array([str(uuid4()) for _ in range(len(detections))]) + inference_ids = np.array([inference_id] * len(detections)) + prediction_type = np.array(["object-detection"] * len(detections)) + detections.data.update( + { + INFERENCE_ID_KEY: inference_ids, + DETECTION_ID_KEY: detection_ids, + PREDICTION_TYPE_KEY: prediction_type, + IMAGE_DIMENSIONS_KEY: dimensions, + } + ) + detections.confidence = np.array([1.0 for _ in detections]) + return attach_parents_coordinates_to_sv_detections( + detections=detections, image=image + ) + + +def get_4digit_from_md5(input_string): + md5_hash = hashlib.md5(input_string.encode("utf-8")) + hex_digest = md5_hash.hexdigest() + integer_value = int(hex_digest[:9], 16) + return integer_value % 10000 + + +REGISTERED_PARSERS = { + ("google-gemini", "object-detection"): parse_gemini_object_detection_response, + ("anthropic-claude", "object-detection"): parse_gemini_object_detection_response, + ("florence-2", "object-detection"): partial( + parse_florence2_object_detection_response, florence_task_type="" + ), + ("florence-2", "open-vocabulary-object-detection"): partial( + parse_florence2_object_detection_response, + florence_task_type="", + ), + ("florence-2", "object-detection-and-caption"): partial( + parse_florence2_object_detection_response, + florence_task_type="", + ), + ("florence-2", "phrase-grounded-object-detection"): partial( + parse_florence2_object_detection_response, + florence_task_type="", + ), + ("florence-2", "region-proposal"): partial( + parse_florence2_object_detection_response, + florence_task_type="", + ), + ("florence-2", "ocr-with-text-detection"): partial( + parse_florence2_object_detection_response, + florence_task_type="", + ), +} diff --git a/inference/core/workflows/core_steps/fusion/detections_classes_replacement/v1.py b/inference/core/workflows/core_steps/fusion/detections_classes_replacement/v1.py index 26bfb287e..038cd6940 100644 --- a/inference/core/workflows/core_steps/fusion/detections_classes_replacement/v1.py +++ b/inference/core/workflows/core_steps/fusion/detections_classes_replacement/v1.py @@ -19,7 +19,7 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -54,7 +54,7 @@ class BlockManifest(WorkflowBlockManifest): "roboflow_core/detections_classes_replacement@v1", "DetectionsClassesReplacement", ] - object_detection_predictions: StepOutputSelector( + object_detection_predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -65,9 +65,7 @@ class BlockManifest(WorkflowBlockManifest): description="The output of a detection model describing the bounding boxes that will have classes replaced.", examples=["$steps.my_object_detection_model.predictions"], ) - classification_predictions: StepOutputSelector( - kind=[CLASSIFICATION_PREDICTION_KIND] - ) = Field( + classification_predictions: Selector(kind=[CLASSIFICATION_PREDICTION_KIND]) = Field( title="Classification results for crops", description="The output of classification model for crops taken based on RoIs pointed as the other parameter", examples=["$steps.my_classification_model.predictions"], @@ -103,7 +101,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DetectionsClassesReplacementBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/fusion/detections_consensus/v1.py b/inference/core/workflows/core_steps/fusion/detections_consensus/v1.py index c3ef35aff..a22965eb2 100644 --- a/inference/core/workflows/core_steps/fusion/detections_consensus/v1.py +++ b/inference/core/workflows/core_steps/fusion/detections_consensus/v1.py @@ -36,8 +36,7 @@ LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, FloatZeroToOne, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -81,7 +80,7 @@ class BlockManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/detections_consensus@v1", "DetectionsConsensus"] predictions_batches: List[ - StepOutputSelector( + Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -94,33 +93,29 @@ class BlockManifest(WorkflowBlockManifest): examples=[["$steps.a.predictions", "$steps.b.predictions"]], validation_alias=AliasChoices("predictions_batches", "predictions"), ) - required_votes: Union[ - PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) - ] = Field( + required_votes: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( description="Required number of votes for single detection from different models to accept detection as output detection", examples=[2, "$inputs.required_votes"], ) - class_aware: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( + class_aware: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=True, description="Flag to decide if merging detections is class-aware or only bounding boxes aware", examples=[True, "$inputs.class_aware"], ) - iou_threshold: Union[ - FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]) - ] = Field( - default=0.3, - description="IoU threshold to consider detections from different models as matching (increasing votes for region)", - examples=[0.3, "$inputs.iou_threshold"], + iou_threshold: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = ( + Field( + default=0.3, + description="IoU threshold to consider detections from different models as matching (increasing votes for region)", + examples=[0.3, "$inputs.iou_threshold"], + ) ) - confidence: Union[ - FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]) - ] = Field( + confidence: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( default=0.0, description="Confidence threshold for merged detections", examples=[0.1, "$inputs.confidence"], ) classes_to_consider: Optional[ - Union[List[str], WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] + Union[List[str], Selector(kind=[LIST_OF_VALUES_KIND])] ] = Field( default=None, description="Optional list of classes to consider in consensus procedure.", @@ -130,7 +125,7 @@ class BlockManifest(WorkflowBlockManifest): Union[ PositiveInt, Dict[str, PositiveInt], - WorkflowParameterSelector(kind=[INTEGER_KIND, DICTIONARY_KIND]), + Selector(kind=[INTEGER_KIND, DICTIONARY_KIND]), ] ] = Field( default=None, @@ -154,8 +149,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["predictions_batches"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -175,7 +170,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DetectionsConsensusBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/fusion/detections_stitch/v1.py b/inference/core/workflows/core_steps/fusion/detections_stitch/v1.py index 1221e7f65..375c8ae39 100644 --- a/inference/core/workflows/core_steps/fusion/detections_stitch/v1.py +++ b/inference/core/workflows/core_steps/fusion/detections_stitch/v1.py @@ -20,14 +20,12 @@ ) from inference.core.workflows.execution_engine.entities.types import ( FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, FloatZeroToOne, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -59,11 +57,11 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/detections_stitch@v1"] - reference_image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + reference_image: Selector(kind=[IMAGE_KIND]) = Field( description="Image that was origin to take crops that yielded predictions.", examples=["$inputs.image"], ) - predictions: StepOutputSelector( + predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -74,7 +72,7 @@ class BlockManifest(WorkflowBlockManifest): ) overlap_filtering_strategy: Union[ Literal["none", "nms", "nmm"], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( default="nms", description="Which strategy to employ when filtering overlapping boxes. " @@ -83,7 +81,7 @@ class BlockManifest(WorkflowBlockManifest): ) iou_threshold: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.3, description="Parameter of overlap filtering strategy. If box intersection over union is above this " @@ -113,7 +111,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DetectionsStitchBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/fusion/dimension_collapse/v1.py b/inference/core/workflows/core_steps/fusion/dimension_collapse/v1.py index a9932f996..c38b9c5df 100644 --- a/inference/core/workflows/core_steps/fusion/dimension_collapse/v1.py +++ b/inference/core/workflows/core_steps/fusion/dimension_collapse/v1.py @@ -8,7 +8,7 @@ ) from inference.core.workflows.execution_engine.entities.types import ( LIST_OF_VALUES_KIND, - StepOutputSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -42,7 +42,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/dimension_collapse@v1", "DimensionCollapse"] - data: StepOutputSelector() = Field( + data: Selector() = Field( description="Reference to step outputs at depth level n to be concatenated and moved into level n-1.", examples=["$steps.ocr_step.results"], ) @@ -64,7 +64,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DimensionCollapseBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/loader.py b/inference/core/workflows/core_steps/loader.py index 140b0ced6..3ffd659d5 100644 --- a/inference/core/workflows/core_steps/loader.py +++ b/inference/core/workflows/core_steps/loader.py @@ -68,7 +68,32 @@ from inference.core.workflows.core_steps.classical_cv.threshold.v1 import ( ImageThresholdBlockV1, ) +from inference.core.workflows.core_steps.common.deserializers import ( + deserialize_boolean_kind, + deserialize_bytes_kind, + deserialize_classification_prediction_kind, + deserialize_detections_kind, + deserialize_dictionary_kind, + deserialize_float_kind, + deserialize_float_zero_to_one_kind, + deserialize_image_kind, + deserialize_integer_kind, + deserialize_list_of_values_kind, + deserialize_numpy_array, + deserialize_optional_string_kind, + deserialize_point_kind, + deserialize_rgb_color_kind, + deserialize_string_kind, + deserialize_video_metadata_kind, + deserialize_zone_kind, +) from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.core_steps.common.serializers import ( + serialise_image, + serialise_sv_detections, + serialize_video_metadata_kind, + serialize_wildcard_kind, +) from inference.core.workflows.core_steps.flow_control.continue_if.v1 import ( ContinueIfBlockV1, ) @@ -91,9 +116,15 @@ from inference.core.workflows.core_steps.formatters.vlm_as_classifier.v1 import ( VLMAsClassifierBlockV1, ) +from inference.core.workflows.core_steps.formatters.vlm_as_classifier.v2 import ( + VLMAsClassifierBlockV2, +) from inference.core.workflows.core_steps.formatters.vlm_as_detector.v1 import ( VLMAsDetectorBlockV1, ) +from inference.core.workflows.core_steps.formatters.vlm_as_detector.v2 import ( + VLMAsDetectorBlockV2, +) from inference.core.workflows.core_steps.fusion.detections_classes_replacement.v1 import ( DetectionsClassesReplacementBlockV1, ) @@ -121,6 +152,9 @@ from inference.core.workflows.core_steps.models.foundation.florence2.v1 import ( Florence2BlockV1, ) +from inference.core.workflows.core_steps.models.foundation.florence2.v2 import ( + Florence2BlockV2, +) from inference.core.workflows.core_steps.models.foundation.google_gemini.v1 import ( GoogleGeminiBlockV1, ) @@ -150,18 +184,33 @@ from inference.core.workflows.core_steps.models.roboflow.instance_segmentation.v1 import ( RoboflowInstanceSegmentationModelBlockV1, ) +from inference.core.workflows.core_steps.models.roboflow.instance_segmentation.v2 import ( + RoboflowInstanceSegmentationModelBlockV2, +) from inference.core.workflows.core_steps.models.roboflow.keypoint_detection.v1 import ( RoboflowKeypointDetectionModelBlockV1, ) +from inference.core.workflows.core_steps.models.roboflow.keypoint_detection.v2 import ( + RoboflowKeypointDetectionModelBlockV2, +) from inference.core.workflows.core_steps.models.roboflow.multi_class_classification.v1 import ( RoboflowClassificationModelBlockV1, ) +from inference.core.workflows.core_steps.models.roboflow.multi_class_classification.v2 import ( + RoboflowClassificationModelBlockV2, +) from inference.core.workflows.core_steps.models.roboflow.multi_label_classification.v1 import ( RoboflowMultiLabelClassificationModelBlockV1, ) +from inference.core.workflows.core_steps.models.roboflow.multi_label_classification.v2 import ( + RoboflowMultiLabelClassificationModelBlockV2, +) from inference.core.workflows.core_steps.models.roboflow.object_detection.v1 import ( RoboflowObjectDetectionModelBlockV1, ) +from inference.core.workflows.core_steps.models.roboflow.object_detection.v2 import ( + RoboflowObjectDetectionModelBlockV2, +) from inference.core.workflows.core_steps.models.third_party.barcode_detection.v1 import ( BarcodeDetectorBlockV1, ) @@ -311,6 +360,7 @@ IMAGE_KEYPOINTS_KIND, IMAGE_KIND, IMAGE_METADATA_KIND, + INFERENCE_ID_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, @@ -346,6 +396,47 @@ "allowed_write_directory": WORKFLOW_BLOCKS_WRITE_DIRECTORY, } +KINDS_SERIALIZERS = { + IMAGE_KIND.name: serialise_image, + VIDEO_METADATA_KIND.name: serialize_video_metadata_kind, + OBJECT_DETECTION_PREDICTION_KIND.name: serialise_sv_detections, + INSTANCE_SEGMENTATION_PREDICTION_KIND.name: serialise_sv_detections, + KEYPOINT_DETECTION_PREDICTION_KIND.name: serialise_sv_detections, + QR_CODE_DETECTION_KIND.name: serialise_sv_detections, + BAR_CODE_DETECTION_KIND.name: serialise_sv_detections, + WILDCARD_KIND.name: serialize_wildcard_kind, +} +KINDS_DESERIALIZERS = { + IMAGE_KIND.name: deserialize_image_kind, + VIDEO_METADATA_KIND.name: deserialize_video_metadata_kind, + OBJECT_DETECTION_PREDICTION_KIND.name: deserialize_detections_kind, + INSTANCE_SEGMENTATION_PREDICTION_KIND.name: deserialize_detections_kind, + KEYPOINT_DETECTION_PREDICTION_KIND.name: deserialize_detections_kind, + QR_CODE_DETECTION_KIND.name: deserialize_detections_kind, + BAR_CODE_DETECTION_KIND.name: deserialize_detections_kind, + NUMPY_ARRAY_KIND.name: deserialize_numpy_array, + ROBOFLOW_MODEL_ID_KIND.name: deserialize_string_kind, + ROBOFLOW_PROJECT_KIND.name: deserialize_string_kind, + ROBOFLOW_API_KEY_KIND.name: deserialize_optional_string_kind, + FLOAT_ZERO_TO_ONE_KIND.name: deserialize_float_zero_to_one_kind, + LIST_OF_VALUES_KIND.name: deserialize_list_of_values_kind, + BOOLEAN_KIND.name: deserialize_boolean_kind, + INTEGER_KIND.name: deserialize_integer_kind, + STRING_KIND.name: deserialize_string_kind, + TOP_CLASS_KIND.name: deserialize_string_kind, + FLOAT_KIND.name: deserialize_float_kind, + DICTIONARY_KIND.name: deserialize_dictionary_kind, + CLASSIFICATION_PREDICTION_KIND.name: deserialize_classification_prediction_kind, + POINT_KIND.name: deserialize_point_kind, + ZONE_KIND.name: deserialize_zone_kind, + RGB_COLOR_KIND.name: deserialize_rgb_color_kind, + LANGUAGE_MODEL_OUTPUT_KIND.name: deserialize_string_kind, + PREDICTION_TYPE_KIND.name: deserialize_string_kind, + PARENT_ID_KIND.name: deserialize_string_kind, + BYTES_KIND.name: deserialize_bytes_kind, + INFERENCE_ID_KIND.name: deserialize_string_kind, +} + def load_blocks() -> List[Type[WorkflowBlock]]: return [ @@ -390,6 +481,7 @@ def load_blocks() -> List[Type[WorkflowBlock]]: DotVisualizationBlockV1, EllipseVisualizationBlockV1, Florence2BlockV1, + Florence2BlockV2, GoogleGeminiBlockV1, GoogleVisionOCRBlockV1, HaloVisualizationBlockV1, @@ -449,6 +541,13 @@ def load_blocks() -> List[Type[WorkflowBlock]]: ReferencePathVisualizationBlockV1, ByteTrackerBlockV3, WebhookSinkBlockV1, + RoboflowInstanceSegmentationModelBlockV2, + RoboflowKeypointDetectionModelBlockV2, + RoboflowClassificationModelBlockV2, + RoboflowMultiLabelClassificationModelBlockV2, + RoboflowObjectDetectionModelBlockV2, + VLMAsClassifierBlockV2, + VLMAsDetectorBlockV2, ] @@ -487,4 +586,5 @@ def load_kinds() -> List[Kind]: PARENT_ID_KIND, IMAGE_METADATA_KIND, BYTES_KIND, + INFERENCE_ID_KIND, ] diff --git a/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v1.py b/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v1.py index 3b42cf520..69a229c6d 100644 --- a/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/anthropic_claude/v1.py @@ -21,14 +21,13 @@ ) from inference.core.workflows.execution_engine.entities.types import ( FLOAT_KIND, + IMAGE_KIND, INTEGER_KIND, LANGUAGE_MODEL_OUTPUT_KIND, LIST_OF_VALUES_KIND, STRING_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -95,10 +94,11 @@ class BlockManifest(WorkflowBlockManifest): "search_keywords": ["LMM", "VLM", "Claude", "Anthropic"], "is_vlm_block": True, "task_type_property": "task_type", - } + }, + protected_namespaces=(), ) type: Literal["roboflow_core/anthropic_claude@v1"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField + images: Selector(kind=[IMAGE_KIND]) = ImageInputField task_type: TaskType = Field( default="unconstrained", description="Task type to be performed by model. Value determines required parameters and output response.", @@ -113,7 +113,7 @@ class BlockManifest(WorkflowBlockManifest): "always_visible": True, }, ) - prompt: Optional[Union[WorkflowParameterSelector(kind=[STRING_KIND]), str]] = Field( + prompt: Optional[Union[Selector(kind=[STRING_KIND]), str]] = Field( default=None, description="Text prompt to the Claude model", examples=["my prompt", "$inputs.prompt"], @@ -136,9 +136,7 @@ class BlockManifest(WorkflowBlockManifest): }, }, ) - classes: Optional[ - Union[WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), List[str]] - ] = Field( + classes: Optional[Union[Selector(kind=[LIST_OF_VALUES_KIND]), List[str]]] = Field( default=None, description="List of classes to be used", examples=[["class-a", "class-b"], "$inputs.classes"], @@ -151,13 +149,13 @@ class BlockManifest(WorkflowBlockManifest): }, }, ) - api_key: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + api_key: Union[Selector(kind=[STRING_KIND]), str] = Field( description="Your Antropic API key", examples=["xxx-xxx", "$inputs.antropics_api_key"], private=True, ) model_version: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), Literal[ "claude-3-5-sonnet", "claude-3-opus", "claude-3-sonnet", "claude-3-haiku" ], @@ -170,16 +168,14 @@ class BlockManifest(WorkflowBlockManifest): default=450, description="Maximum number of tokens the model can generate in it's response.", ) - temperature: Optional[ - Union[float, WorkflowParameterSelector(kind=[FLOAT_KIND])] - ] = Field( + temperature: Optional[Union[float, Selector(kind=[FLOAT_KIND])]] = Field( default=None, description="Temperature to sample from the model - value in range 0.0-2.0, the higher - the more " 'random / "creative" the generations are.', ge=0.0, le=2.0, ) - max_image_size: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( + max_image_size: Union[int, Selector(kind=[INTEGER_KIND])] = Field( description="Maximum size of the image - if input has larger side, it will be downscaled, keeping aspect ratio", default=1024, ) @@ -210,8 +206,8 @@ def validate(self) -> "BlockManifest": return self @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -224,7 +220,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class AntropicClaudeBlockV1(WorkflowBlock): @@ -247,7 +243,7 @@ def get_manifest(cls) -> Type[WorkflowBlockManifest]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" def run( self, diff --git a/inference/core/workflows/core_steps/models/foundation/clip_comparison/v1.py b/inference/core/workflows/core_steps/models/foundation/clip_comparison/v1.py index f8cb0a355..4118b969c 100644 --- a/inference/core/workflows/core_steps/models/foundation/clip_comparison/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/clip_comparison/v1.py @@ -28,13 +28,12 @@ WorkflowImageData, ) from inference.core.workflows.execution_engine.entities.types import ( + IMAGE_KIND, LIST_OF_VALUES_KIND, PARENT_ID_KIND, PREDICTION_TYPE_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -70,18 +69,16 @@ class BlockManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/clip_comparison@v1", "ClipComparison"] name: str = Field(description="Unique name of step in workflows") - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - texts: Union[WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), List[str]] = ( - Field( - description="List of texts to calculate similarity against each input image", - examples=[["a", "b", "c"], "$inputs.texts"], - validation_alias=AliasChoices("texts", "text"), - ) + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + texts: Union[Selector(kind=[LIST_OF_VALUES_KIND]), List[str]] = Field( + description="List of texts to calculate similarity against each input image", + examples=[["a", "b", "c"], "$inputs.texts"], + validation_alias=AliasChoices("texts", "text"), ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -94,7 +91,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ClipComparisonBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/models/foundation/clip_comparison/v2.py b/inference/core/workflows/core_steps/models/foundation/clip_comparison/v2.py index 165b020cc..a1ee4b52c 100644 --- a/inference/core/workflows/core_steps/models/foundation/clip_comparison/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/clip_comparison/v2.py @@ -29,13 +29,12 @@ from inference.core.workflows.execution_engine.entities.types import ( CLASSIFICATION_PREDICTION_KIND, FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, LIST_OF_VALUES_KIND, PARENT_ID_KIND, STRING_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -69,13 +68,11 @@ class BlockManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/clip_comparison@v2"] name: str = Field(description="Unique name of step in workflows") - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - classes: Union[WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), List[str]] = ( - Field( - description="List of classes to calculate similarity against each input image", - examples=[["a", "b", "c"], "$inputs.texts"], - min_items=1, - ) + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + classes: Union[Selector(kind=[LIST_OF_VALUES_KIND]), List[str]] = Field( + description="List of classes to calculate similarity against each input image", + examples=[["a", "b", "c"], "$inputs.texts"], + min_items=1, ) version: Union[ Literal[ @@ -89,7 +86,7 @@ class BlockManifest(WorkflowBlockManifest): "ViT-L-14-336px", "ViT-L-14", ], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( default="ViT-B-16", description="Variant of CLIP model", @@ -97,8 +94,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -118,7 +115,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ClipComparisonBlockV2(WorkflowBlock): @@ -207,7 +204,7 @@ def run_remotely( tasks = [ partial( client.clip_compare, - subject=single_image.numpy_image, + subject=single_image.base64_image, prompt=classes, clip_version=version, ) diff --git a/inference/core/workflows/core_steps/models/foundation/cog_vlm/v1.py b/inference/core/workflows/core_steps/models/foundation/cog_vlm/v1.py index 72e9a59d6..1a6678b11 100644 --- a/inference/core/workflows/core_steps/models/foundation/cog_vlm/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/cog_vlm/v1.py @@ -5,11 +5,7 @@ from pydantic import ConfigDict, Field from inference.core.entities.requests.cogvlm import CogVLMInferenceRequest -from inference.core.env import ( - LOCAL_INFERENCE_API_URL, - WORKFLOWS_REMOTE_API_TARGET, - WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, -) +from inference.core.env import LOCAL_INFERENCE_API_URL, WORKFLOWS_REMOTE_API_TARGET from inference.core.managers.base import ModelManager from inference.core.utils.image_utils import load_image from inference.core.workflows.core_steps.common.entities import StepExecutionMode @@ -25,14 +21,13 @@ ) from inference.core.workflows.execution_engine.entities.types import ( DICTIONARY_KIND, + IMAGE_KIND, IMAGE_METADATA_KIND, PARENT_ID_KIND, STRING_KIND, WILDCARD_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -40,7 +35,6 @@ WorkflowBlockManifest, ) from inference_sdk import InferenceHTTPClient -from inference_sdk.http.utils.iterables import make_batches NOT_DETECTED_VALUE = "not_detected" @@ -68,8 +62,8 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/cog_vlm@v1", "CogVLM"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - prompt: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + prompt: Union[Selector(kind=[STRING_KIND]), str] = Field( description="Text prompt to the CogVLM model", examples=["my prompt", "$inputs.prompt"], ) @@ -83,8 +77,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -113,7 +107,7 @@ def get_actual_outputs(self) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class CogVLMBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/models/foundation/florence2/v1.py b/inference/core/workflows/core_steps/models/foundation/florence2/v1.py index 930977a6a..b42f50456 100644 --- a/inference/core/workflows/core_steps/models/foundation/florence2/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/florence2/v1.py @@ -16,6 +16,7 @@ ) from inference.core.workflows.execution_engine.entities.types import ( DICTIONARY_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, LANGUAGE_MODEL_OUTPUT_KIND, @@ -23,10 +24,7 @@ OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, ImageInputField, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -37,6 +35,14 @@ T = TypeVar("T") K = TypeVar("K") +FLORENCE_TASKS_METADATA = { + "custom": { + "name": "Custom Prompt", + "description": "Use free-form prompt to generate a response. Useful with finetuned models.", + }, + **VLM_TASKS_METADATA, +} + DETECTIONS_CLASS_NAME_FIELD = "class_name" DETECTION_ID_FIELD = "detection_id" @@ -77,12 +83,13 @@ }, {"task_type": "detection-grounded-ocr", "florence_task": ""}, {"task_type": "region-proposal", "florence_task": ""}, + {"task_type": "custom", "florence_task": None}, ] TASK_TYPE_TO_FLORENCE_TASK = { task["task_type"]: task["florence_task"] for task in SUPPORTED_TASK_TYPES_LIST } RELEVANT_TASKS_METADATA = { - k: v for k, v in VLM_TASKS_METADATA.items() if k in TASK_TYPE_TO_FLORENCE_TASK + k: v for k, v in FLORENCE_TASKS_METADATA.items() if k in TASK_TYPE_TO_FLORENCE_TASK } RELEVANT_TASKS_DOCS_DESCRIPTION = "\n\n".join( f"* **{v['name']}** (`{k}`) - {v['description']}" @@ -127,6 +134,7 @@ TASKS_REQUIRING_PROMPT = { "phrase-grounded-object-detection", "phrase-grounded-instance-segmentation", + "custom", } TASKS_REQUIRING_CLASSES = { "open-vocabulary-object-detection", @@ -147,31 +155,8 @@ } -class BlockManifest(WorkflowBlockManifest): - model_config = ConfigDict( - json_schema_extra={ - "name": "Florence-2 Model", - "version": "v1", - "short_description": "Run Florence-2 on an image", - "long_description": LONG_DESCRIPTION, - "license": "Apache-2.0", - "block_type": "model", - "search_keywords": ["Florence", "Florence-2", "Microsoft"], - "is_vlm_block": True, - "task_type_property": "task_type", - }, - protected_namespaces=(), - ) - type: Literal["roboflow_core/florence_2@v1"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - model_version: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), - Literal["florence-2-base", "florence-2-large"], - ] = Field( - default="florence-2-base", - description="Model to be used", - examples=["florence-2-base"], - ) +class BaseManifest(WorkflowBlockManifest): + images: Selector(kind=[IMAGE_KIND]) = ImageInputField task_type: TaskType = Field( default="open-vocabulary-object-detection", description="Task type to be performed by model. " @@ -189,7 +174,7 @@ class BlockManifest(WorkflowBlockManifest): "always_visible": True, }, ) - prompt: Optional[Union[WorkflowParameterSelector(kind=[STRING_KIND]), str]] = Field( + prompt: Optional[Union[Selector(kind=[STRING_KIND]), str]] = Field( default=None, description="Text prompt to the Florence-2 model", examples=["my prompt", "$inputs.prompt"], @@ -199,9 +184,7 @@ class BlockManifest(WorkflowBlockManifest): }, }, ) - classes: Optional[ - Union[WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), List[str]] - ] = Field( + classes: Optional[Union[Selector(kind=[LIST_OF_VALUES_KIND]), List[str]]] = Field( default=None, description="List of classes to be used", examples=[["class-a", "class-b"], "$inputs.classes"], @@ -218,14 +201,14 @@ class BlockManifest(WorkflowBlockManifest): Union[ List[int], List[float], - StepOutputSelector( + Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ] ), - WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), + Selector(kind=[LIST_OF_VALUES_KIND]), ] ] = Field( default=None, @@ -257,8 +240,12 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def get_parameters_accepting_batches_and_scalars(cls) -> List[str]: + return ["grounding_detection"] @model_validator(mode="after") def validate(self) -> "BlockManifest": @@ -291,7 +278,33 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" + + +class BlockManifest(BaseManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Florence-2 Model", + "version": "v1", + "short_description": "Run Florence-2 on an image", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + "search_keywords": ["Florence", "Florence-2", "Microsoft"], + "is_vlm_block": True, + "task_type_property": "task_type", + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/florence_2@v1"] + model_version: Union[ + Selector(kind=[STRING_KIND]), + Literal["florence-2-base", "florence-2-large"], + ] = Field( + default="florence-2-base", + description="Model to be used", + examples=["florence-2-base"], + ) class Florence2BlockV1(WorkflowBlock): @@ -358,6 +371,8 @@ def run_locally( grounding_selection_mode: GroundingSelectionMode, ) -> BlockResult: requires_detection_grounding = task_type in TASKS_REQUIRING_DETECTION_GROUNDING + + is_not_florence_task = task_type == "custom" task_type = TASK_TYPE_TO_FLORENCE_TASK[task_type] inference_images = [ i.to_inference_format(numpy_preferred=False) for i in images @@ -385,17 +400,27 @@ def run_locally( {"raw_output": None, "parsed_output": None, "classes": None} ) continue + if is_not_florence_task: + prompt = single_prompt or "" + else: + prompt = task_type + (single_prompt or "") + request = LMMInferenceRequest( api_key=self._api_key, model_id=model_version, image=image, source="workflow-execution", - prompt=task_type + (single_prompt or ""), + prompt=prompt, ) prediction = self._model_manager.infer_from_request_sync( model_id=model_version, request=request ) - prediction_data = prediction.response[task_type] + if is_not_florence_task: + prediction_data = prediction.response[ + list(prediction.response.keys())[0] + ] + else: + prediction_data = prediction.response[task_type] if task_type in TASKS_TO_EXTRACT_LABELS_AS_CLASSES: classes = prediction_data.get("labels", []) predictions.append( diff --git a/inference/core/workflows/core_steps/models/foundation/florence2/v2.py b/inference/core/workflows/core_steps/models/foundation/florence2/v2.py new file mode 100644 index 000000000..28a11e248 --- /dev/null +++ b/inference/core/workflows/core_steps/models/foundation/florence2/v2.py @@ -0,0 +1,75 @@ +from typing import List, Literal, Optional, Type, Union + +import supervision as sv +from pydantic import ConfigDict, Field + +from inference.core.workflows.core_steps.models.foundation.florence2.v1 import ( + LONG_DESCRIPTION, + BaseManifest, + Florence2BlockV1, + GroundingSelectionMode, + TaskType, +) +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + ROBOFLOW_MODEL_ID_KIND, + WorkflowParameterSelector, +) +from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest + + +class V2BlockManifest(BaseManifest): + type: Literal["roboflow_core/florence_2@v2"] + model_id: Union[WorkflowParameterSelector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = ( + Field( + default="florence-2-base", + description="Model to be used", + examples=["florence-2-base"], + json_schema_extra={"always_visible": True}, + ) + ) + model_config = ConfigDict( + json_schema_extra={ + "name": "Florence-2 Model", + "version": "v2", + "short_description": "Run Florence-2 on an image", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + "search_keywords": ["Florence", "Florence-2", "Microsoft"], + "is_vlm_block": True, + "task_type_property": "task_type", + }, + protected_namespaces=(), + ) + + +class Florence2BlockV2(Florence2BlockV1): + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return V2BlockManifest + + def run( + self, + images: Batch[WorkflowImageData], + model_id: str, + task_type: TaskType, + prompt: Optional[str], + classes: Optional[List[str]], + grounding_detection: Optional[ + Union[Batch[sv.Detections], List[int], List[float]] + ], + grounding_selection_mode: GroundingSelectionMode, + ) -> BlockResult: + return super().run( + images=images, + model_version=model_id, + task_type=task_type, + prompt=prompt, + classes=classes, + grounding_detection=grounding_detection, + grounding_selection_mode=grounding_selection_mode, + ) diff --git a/inference/core/workflows/core_steps/models/foundation/google_gemini/v1.py b/inference/core/workflows/core_steps/models/foundation/google_gemini/v1.py index 4f7a6285c..99d4e4608 100644 --- a/inference/core/workflows/core_steps/models/foundation/google_gemini/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/google_gemini/v1.py @@ -20,13 +20,12 @@ ) from inference.core.workflows.execution_engine.entities.types import ( FLOAT_KIND, + IMAGE_KIND, LANGUAGE_MODEL_OUTPUT_KIND, LIST_OF_VALUES_KIND, STRING_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -104,10 +103,11 @@ class BlockManifest(WorkflowBlockManifest): "beta": True, "is_vlm_block": True, "task_type_property": "task_type", - } + }, + protected_namespaces=(), ) type: Literal["roboflow_core/google_gemini@v1"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField + images: Selector(kind=[IMAGE_KIND]) = ImageInputField task_type: TaskType = Field( default="unconstrained", description="Task type to be performed by model. Value determines required parameters and output response.", @@ -122,7 +122,7 @@ class BlockManifest(WorkflowBlockManifest): "always_visible": True, }, ) - prompt: Optional[Union[WorkflowParameterSelector(kind=[STRING_KIND]), str]] = Field( + prompt: Optional[Union[Selector(kind=[STRING_KIND]), str]] = Field( default=None, description="Text prompt to the Gemini model", examples=["my prompt", "$inputs.prompt"], @@ -145,9 +145,7 @@ class BlockManifest(WorkflowBlockManifest): }, }, ) - classes: Optional[ - Union[WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), List[str]] - ] = Field( + classes: Optional[Union[Selector(kind=[LIST_OF_VALUES_KIND]), List[str]]] = Field( default=None, description="List of classes to be used", examples=[["class-a", "class-b"], "$inputs.classes"], @@ -160,13 +158,13 @@ class BlockManifest(WorkflowBlockManifest): }, }, ) - api_key: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + api_key: Union[Selector(kind=[STRING_KIND]), str] = Field( description="Your Google AI API key", examples=["xxx-xxx", "$inputs.google_api_key"], private=True, ) model_version: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), Literal["gemini-1.5-flash", "gemini-1.5-pro"], ] = Field( default="gemini-1.5-flash", @@ -177,9 +175,7 @@ class BlockManifest(WorkflowBlockManifest): default=450, description="Maximum number of tokens the model can generate in it's response.", ) - temperature: Optional[ - Union[float, WorkflowParameterSelector(kind=[FLOAT_KIND])] - ] = Field( + temperature: Optional[Union[float, Selector(kind=[FLOAT_KIND])]] = Field( default=None, description="Temperature to sample from the model - value in range 0.0-2.0, the higher - the more " 'random / "creative" the generations are.', @@ -213,8 +209,8 @@ def validate(self) -> "BlockManifest": return self @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -227,7 +223,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class GoogleGeminiBlockV1(WorkflowBlock): @@ -250,7 +246,7 @@ def get_manifest(cls) -> Type[WorkflowBlockManifest]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" def run( self, diff --git a/inference/core/workflows/core_steps/models/foundation/google_vision_ocr/v1.py b/inference/core/workflows/core_steps/models/foundation/google_vision_ocr/v1.py index b24b5633a..5fa0158e1 100644 --- a/inference/core/workflows/core_steps/models/foundation/google_vision_ocr/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/google_vision_ocr/v1.py @@ -20,11 +20,10 @@ WorkflowImageData, ) from inference.core.workflows.execution_engine.entities.types import ( + IMAGE_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -61,7 +60,7 @@ class BlockManifest(WorkflowBlockManifest): protected_namespaces=(), ) type: Literal["roboflow_core/google_vision_ocr@v1"] - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Image to run OCR", examples=["$inputs.image", "$steps.cropping.crops"], ) @@ -80,7 +79,7 @@ class BlockManifest(WorkflowBlockManifest): }, }, ) - api_key: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + api_key: Union[Selector(kind=[STRING_KIND]), str] = Field( description="Your Google Vision API key", examples=["xxx-xxx", "$inputs.google_api_key"], private=True, @@ -98,7 +97,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class GoogleVisionOCRBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/models/foundation/lmm/v1.py b/inference/core/workflows/core_steps/models/foundation/lmm/v1.py index cd8063363..0234c3b75 100644 --- a/inference/core/workflows/core_steps/models/foundation/lmm/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/lmm/v1.py @@ -31,14 +31,13 @@ ) from inference.core.workflows.execution_engine.entities.types import ( DICTIONARY_KIND, + IMAGE_KIND, IMAGE_METADATA_KIND, PARENT_ID_KIND, STRING_KIND, WILDCARD_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -94,14 +93,12 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/lmm@v1", "LMM"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - prompt: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + prompt: Union[Selector(kind=[STRING_KIND]), str] = Field( description="Holds unconstrained text prompt to LMM mode", examples=["my prompt", "$inputs.prompt"], ) - lmm_type: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), Literal["gpt_4v", "cog_vlm"] - ] = Field( + lmm_type: Union[Selector(kind=[STRING_KIND]), Literal["gpt_4v", "cog_vlm"]] = Field( description="Type of LMM to be used", examples=["gpt_4v", "$inputs.lmm_type"] ) lmm_config: LMMConfig = Field( @@ -115,9 +112,7 @@ class BlockManifest(WorkflowBlockManifest): } ], ) - remote_api_key: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), Optional[str] - ] = Field( + remote_api_key: Union[Selector(kind=[STRING_KIND]), Optional[str]] = Field( default=None, description="Holds API key required to call LMM model - in current state of development, we require OpenAI key when `lmm_type=gpt_4v` and do not require additional API key for CogVLM calls.", examples=["xxx-xxx", "$inputs.api_key"], @@ -130,8 +125,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -160,7 +155,7 @@ def get_actual_outputs(self) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class LMMBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/models/foundation/lmm_classifier/v1.py b/inference/core/workflows/core_steps/models/foundation/lmm_classifier/v1.py index 3f468a332..fe3d8ee33 100644 --- a/inference/core/workflows/core_steps/models/foundation/lmm_classifier/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/lmm_classifier/v1.py @@ -23,6 +23,7 @@ WorkflowImageData, ) from inference.core.workflows.execution_engine.entities.types import ( + IMAGE_KIND, IMAGE_METADATA_KIND, LIST_OF_VALUES_KIND, PARENT_ID_KIND, @@ -30,9 +31,7 @@ STRING_KIND, TOP_CLASS_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -68,17 +67,13 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/lmm_for_classification@v1", "LMMForClassification"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - lmm_type: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), Literal["gpt_4v", "cog_vlm"] - ] = Field( + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + lmm_type: Union[Selector(kind=[STRING_KIND]), Literal["gpt_4v", "cog_vlm"]] = Field( description="Type of LMM to be used", examples=["gpt_4v", "$inputs.lmm_type"] ) - classes: Union[List[str], WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] = ( - Field( - description="List of classes that LMM shall classify against", - examples=[["a", "b"], "$inputs.classes"], - ) + classes: Union[List[str], Selector(kind=[LIST_OF_VALUES_KIND])] = Field( + description="List of classes that LMM shall classify against", + examples=[["a", "b"], "$inputs.classes"], ) lmm_config: LMMConfig = Field( default_factory=lambda: LMMConfig(), @@ -91,9 +86,7 @@ class BlockManifest(WorkflowBlockManifest): } ], ) - remote_api_key: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), Optional[str] - ] = Field( + remote_api_key: Union[Selector(kind=[STRING_KIND]), Optional[str]] = Field( default=None, description="Holds API key required to call LMM model - in current state of development, we require OpenAI key when `lmm_type=gpt_4v` and do not require additional API key for CogVLM calls.", examples=["xxx-xxx", "$inputs.api_key"], @@ -101,8 +94,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -117,7 +110,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class LMMForClassificationBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/models/foundation/ocr/v1.py b/inference/core/workflows/core_steps/models/foundation/ocr/v1.py index 0b98c263d..b560032e8 100644 --- a/inference/core/workflows/core_steps/models/foundation/ocr/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/ocr/v1.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional, Type, Union +from typing import List, Literal, Optional, Type from pydantic import ConfigDict, Field @@ -27,12 +27,12 @@ WorkflowImageData, ) from inference.core.workflows.execution_engine.entities.types import ( + IMAGE_KIND, PARENT_ID_KIND, PREDICTION_TYPE_KIND, STRING_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -71,11 +71,11 @@ class BlockManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/ocr_model@v1", "OCRModel"] name: str = Field(description="Unique name of step in workflows") - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField + images: Selector(kind=[IMAGE_KIND]) = ImageInputField @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -88,7 +88,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class OCRModelBlockV1(WorkflowBlock): @@ -169,7 +169,7 @@ def run_remotely( max_concurrent_requests=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, ) client.configure(configuration) - non_empty_inference_images = [i.numpy_image for i in images] + non_empty_inference_images = [i.base64_image for i in images] predictions = client.ocr_image( inference_input=non_empty_inference_images, ) diff --git a/inference/core/workflows/core_steps/models/foundation/openai/v1.py b/inference/core/workflows/core_steps/models/foundation/openai/v1.py index 78defbe28..d9f10d170 100644 --- a/inference/core/workflows/core_steps/models/foundation/openai/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/openai/v1.py @@ -22,21 +22,19 @@ ) from inference.core.workflows.execution_engine.entities.types import ( DICTIONARY_KIND, + IMAGE_KIND, IMAGE_METADATA_KIND, PARENT_ID_KIND, STRING_KIND, WILDCARD_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, WorkflowBlock, WorkflowBlockManifest, ) -from inference_sdk.http.utils.iterables import make_batches NOT_DETECTED_VALUE = "not_detected" JSON_MARKDOWN_BLOCK_PATTERN = re.compile(r"```json\n([\s\S]*?)\n```") @@ -72,20 +70,18 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/open_ai@v1", "OpenAI"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - prompt: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + prompt: Union[Selector(kind=[STRING_KIND]), str] = Field( description="Text prompt to the OpenAI model", examples=["my prompt", "$inputs.prompt"], ) - openai_api_key: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), Optional[str] - ] = Field( + openai_api_key: Union[Selector(kind=[STRING_KIND]), Optional[str]] = Field( description="Your OpenAI API key", examples=["xxx-xxx", "$inputs.openai_api_key"], private=True, ) openai_model: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), Literal["gpt-4o", "gpt-4o-mini"] + Selector(kind=[STRING_KIND]), Literal["gpt-4o", "gpt-4o-mini"] ] = Field( default="gpt-4o", description="Model to be used", @@ -100,7 +96,7 @@ class BlockManifest(WorkflowBlockManifest): ], ) image_detail: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), Literal["auto", "high", "low"] + Selector(kind=[STRING_KIND]), Literal["auto", "high", "low"] ] = Field( default="auto", description="Indicates the image's quality, with 'high' suggesting it is of high resolution and should be processed or displayed with high fidelity.", @@ -113,8 +109,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -143,7 +139,7 @@ def get_actual_outputs(self) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class OpenAIBlockV1(WorkflowBlock): @@ -166,7 +162,7 @@ def get_manifest(cls) -> Type[WorkflowBlockManifest]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" def run( self, diff --git a/inference/core/workflows/core_steps/models/foundation/openai/v2.py b/inference/core/workflows/core_steps/models/foundation/openai/v2.py index 0903d9d3a..1f9d03aca 100644 --- a/inference/core/workflows/core_steps/models/foundation/openai/v2.py +++ b/inference/core/workflows/core_steps/models/foundation/openai/v2.py @@ -19,13 +19,12 @@ ) from inference.core.workflows.execution_engine.entities.types import ( FLOAT_KIND, + IMAGE_KIND, LANGUAGE_MODEL_OUTPUT_KIND, LIST_OF_VALUES_KIND, STRING_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -94,10 +93,11 @@ class BlockManifest(WorkflowBlockManifest): "search_keywords": ["LMM", "VLM", "ChatGPT", "GPT", "OpenAI"], "is_vlm_block": True, "task_type_property": "task_type", - } + }, + protected_namespaces=(), ) type: Literal["roboflow_core/open_ai@v2"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField + images: Selector(kind=[IMAGE_KIND]) = ImageInputField task_type: TaskType = Field( default="unconstrained", description="Task type to be performed by model. Value determines required parameters and output response.", @@ -111,7 +111,7 @@ class BlockManifest(WorkflowBlockManifest): "always_visible": True, }, ) - prompt: Optional[Union[WorkflowParameterSelector(kind=[STRING_KIND]), str]] = Field( + prompt: Optional[Union[Selector(kind=[STRING_KIND]), str]] = Field( default=None, description="Text prompt to the OpenAI model", examples=["my prompt", "$inputs.prompt"], @@ -134,9 +134,7 @@ class BlockManifest(WorkflowBlockManifest): }, }, ) - classes: Optional[ - Union[WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), List[str]] - ] = Field( + classes: Optional[Union[Selector(kind=[LIST_OF_VALUES_KIND]), List[str]]] = Field( default=None, description="List of classes to be used", examples=[["class-a", "class-b"], "$inputs.classes"], @@ -149,20 +147,20 @@ class BlockManifest(WorkflowBlockManifest): }, }, ) - api_key: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + api_key: Union[Selector(kind=[STRING_KIND]), str] = Field( description="Your OpenAI API key", examples=["xxx-xxx", "$inputs.openai_api_key"], private=True, ) model_version: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), Literal["gpt-4o", "gpt-4o-mini"] + Selector(kind=[STRING_KIND]), Literal["gpt-4o", "gpt-4o-mini"] ] = Field( default="gpt-4o", description="Model to be used", examples=["gpt-4o", "$inputs.openai_model"], ) image_detail: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), Literal["auto", "high", "low"] + Selector(kind=[STRING_KIND]), Literal["auto", "high", "low"] ] = Field( default="auto", description="Indicates the image's quality, with 'high' suggesting it is of high resolution and should be processed or displayed with high fidelity.", @@ -172,9 +170,7 @@ class BlockManifest(WorkflowBlockManifest): default=450, description="Maximum number of tokens the model can generate in it's response.", ) - temperature: Optional[ - Union[float, WorkflowParameterSelector(kind=[FLOAT_KIND])] - ] = Field( + temperature: Optional[Union[float, Selector(kind=[FLOAT_KIND])]] = Field( default=None, description="Temperature to sample from the model - value in range 0.0-2.0, the higher - the more " 'random / "creative" the generations are.', @@ -208,8 +204,8 @@ def validate(self) -> "BlockManifest": return self @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -222,7 +218,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class OpenAIBlockV2(WorkflowBlock): @@ -245,7 +241,7 @@ def get_manifest(cls) -> Type[WorkflowBlockManifest]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" def run( self, diff --git a/inference/core/workflows/core_steps/models/foundation/segment_anything2/v1.py b/inference/core/workflows/core_steps/models/foundation/segment_anything2/v1.py index c005c20c6..5893248f6 100644 --- a/inference/core/workflows/core_steps/models/foundation/segment_anything2/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/segment_anything2/v1.py @@ -33,15 +33,13 @@ from inference.core.workflows.execution_engine.entities.types import ( BOOLEAN_KIND, FLOAT_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, ImageInputField, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -80,9 +78,9 @@ class BlockManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/segment_anything@v1"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField + images: Selector(kind=[IMAGE_KIND]) = ImageInputField boxes: Optional[ - StepOutputSelector( + Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -96,7 +94,7 @@ class BlockManifest(WorkflowBlockManifest): json_schema_extra={"always_visible": True}, ) version: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), Literal["hiera_large", "hiera_small", "hiera_tiny", "hiera_b_plus"], ] = Field( default="hiera_tiny", @@ -104,23 +102,21 @@ class BlockManifest(WorkflowBlockManifest): examples=["hiera_large", "$inputs.openai_model"], ) threshold: Union[ - WorkflowParameterSelector(kind=[FLOAT_KIND]), + Selector(kind=[FLOAT_KIND]), float, ] = Field( default=0.0, description="Threshold for predicted masks scores", examples=[0.3] ) - multimask_output: Union[ - Optional[bool], WorkflowParameterSelector(kind=[BOOLEAN_KIND]) - ] = Field( + multimask_output: Union[Optional[bool], Selector(kind=[BOOLEAN_KIND])] = Field( default=True, description="Flag to determine whether to use sam2 internal multimask or single mask mode. For ambiguous prompts setting to True is recomended.", examples=[True, "$inputs.multimask_output"], ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images", "boxes"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -133,7 +129,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class SegmentAnything2BlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/models/foundation/stability_ai/inpainting/v1.py b/inference/core/workflows/core_steps/models/foundation/stability_ai/inpainting/v1.py index d5e5e0db7..6f9d87796 100644 --- a/inference/core/workflows/core_steps/models/foundation/stability_ai/inpainting/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/stability_ai/inpainting/v1.py @@ -19,10 +19,7 @@ IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, STRING_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -64,20 +61,18 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/stability_ai_inpainting@v1"] - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="The image which was the base to generate VLM prediction", examples=["$inputs.image", "$steps.cropping.crops"], ) - segmentation_mask: StepOutputSelector( - kind=[INSTANCE_SEGMENTATION_PREDICTION_KIND] - ) = Field( + segmentation_mask: Selector(kind=[INSTANCE_SEGMENTATION_PREDICTION_KIND]) = Field( name="Segmentation Mask", description="Segmentation masks", examples=["$steps.model.predictions"], ) prompt: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), - StepOutputSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), str, ] = Field( description="Prompt to inpainting model (what you wish to see)", @@ -85,8 +80,8 @@ class BlockManifest(WorkflowBlockManifest): ) negative_prompt: Optional[ Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), - StepOutputSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), str, ] ] = Field( @@ -94,7 +89,7 @@ class BlockManifest(WorkflowBlockManifest): description="Negative prompt to inpainting model (what you do not wish to see)", examples=["my prompt", "$inputs.prompt"], ) - api_key: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + api_key: Union[Selector(kind=[STRING_KIND]), str] = Field( description="Your Stability AI API key", examples=["xxx-xxx", "$inputs.stability_ai_api_key"], private=True, @@ -108,7 +103,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class StabilityAIInpaintingBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/models/foundation/yolo_world/v1.py b/inference/core/workflows/core_steps/models/foundation/yolo_world/v1.py index ce9be725c..da3c890f0 100644 --- a/inference/core/workflows/core_steps/models/foundation/yolo_world/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/yolo_world/v1.py @@ -24,14 +24,13 @@ ) from inference.core.workflows.execution_engine.entities.types import ( FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, FloatZeroToOne, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -67,10 +66,8 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/yolo_world_model@v1", "YoloWorldModel", "YoloWorld"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - class_names: Union[ - WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), List[str] - ] = Field( + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + class_names: Union[Selector(kind=[LIST_OF_VALUES_KIND]), List[str]] = Field( description="One or more classes that you want YOLO-World to detect. The model accepts any string as an input, though does best with short descriptions of common objects.", examples=[["person", "car", "license plate"], "$inputs.class_names"], ) @@ -85,7 +82,7 @@ class BlockManifest(WorkflowBlockManifest): "l", "x", ], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( default="v2-s", description="Variant of YoloWorld model", @@ -93,7 +90,7 @@ class BlockManifest(WorkflowBlockManifest): ) confidence: Union[ Optional[FloatZeroToOne], - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.005, description="Confidence threshold for detections", @@ -101,8 +98,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -114,7 +111,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class YoloWorldModelBlockV1(WorkflowBlock): @@ -217,7 +214,7 @@ def run_remotely( client.configure(inference_configuration=configuration) if WORKFLOWS_REMOTE_API_TARGET == "hosted": client.select_api_v0() - inference_images = [i.to_inference_format(numpy_preferred=True) for i in images] + inference_images = [i.to_inference_format() for i in images] image_sub_batches = list( make_batches( iterable=inference_images, diff --git a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v1.py b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v1.py index c3480c4cf..8d529e595 100644 --- a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v1.py @@ -29,6 +29,7 @@ from inference.core.workflows.execution_engine.entities.types import ( BOOLEAN_KIND, FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, LIST_OF_VALUES_KIND, @@ -38,9 +39,7 @@ FloatZeroToOne, ImageInputField, RoboflowModelField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -78,27 +77,23 @@ class BlockManifest(WorkflowBlockManifest): "RoboflowInstanceSegmentationModel", "InstanceSegmentationModel", ] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - model_id: Union[WorkflowParameterSelector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = ( - RoboflowModelField + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + class_agnostic_nms: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=False, + description="Value to decide if NMS is to be used in class-agnostic mode.", + examples=[True, "$inputs.class_agnostic_nms"], ) - class_agnostic_nms: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = ( + class_filter: Union[Optional[List[str]], Selector(kind=[LIST_OF_VALUES_KIND])] = ( Field( - default=False, - description="Value to decide if NMS is to be used in class-agnostic mode.", - examples=[True, "$inputs.class_agnostic_nms"], + default=None, + description="List of classes to retrieve from predictions (to define subset of those which was used while model training)", + examples=[["a", "b", "c"], "$inputs.class_filter"], ) ) - class_filter: Union[ - Optional[List[str]], WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]) - ] = Field( - default=None, - description="List of classes to retrieve from predictions (to define subset of those which was used while model training)", - examples=[["a", "b", "c"], "$inputs.class_filter"], - ) confidence: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Confidence threshold for predictions", @@ -106,29 +101,25 @@ class BlockManifest(WorkflowBlockManifest): ) iou_threshold: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.3, description="Parameter of NMS, to decide on minimum box intersection over union to merge boxes", examples=[0.4, "$inputs.iou_threshold"], ) - max_detections: Union[ - PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) - ] = Field( + max_detections: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( default=300, description="Maximum number of detections to return", examples=[300, "$inputs.max_detections"], ) - max_candidates: Union[ - PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) - ] = Field( + max_candidates: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( default=3000, description="Maximum number of candidates as NMS input to be taken into account.", examples=[3000, "$inputs.max_candidates"], ) mask_decode_mode: Union[ Literal["accurate", "tradeoff", "fast"], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( default="accurate", description="Parameter of mask decoding in prediction post-processing.", @@ -136,21 +127,19 @@ class BlockManifest(WorkflowBlockManifest): ) tradeoff_factor: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.0, description="Post-processing parameter to dictate tradeoff between fast and accurate", examples=[0.3, "$inputs.tradeoff_factor"], ) - disable_active_learning: Union[ - bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND]) - ] = Field( + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=True, description="Parameter to decide if Active Learning data sampling is disabled for the model", examples=[True, "$inputs.disable_active_learning"], ) active_learning_target_dataset: Union[ - WorkflowParameterSelector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] ] = Field( default=None, description="Target dataset for Active Learning data sampling - see Roboflow Active Learning " @@ -159,8 +148,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -174,7 +163,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RoboflowInstanceSegmentationModelBlockV1(WorkflowBlock): @@ -339,7 +328,7 @@ def run_remotely( source="workflow-execution", ) client.configure(inference_configuration=client_config) - inference_images = [i.numpy_image for i in images] + inference_images = [i.base64_image for i in images] predictions = client.infer( inference_input=inference_images, model_id=model_id, diff --git a/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v2.py b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v2.py new file mode 100644 index 000000000..a7972dfe4 --- /dev/null +++ b/inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v2.py @@ -0,0 +1,364 @@ +from typing import List, Literal, Optional, Type, Union + +from pydantic import ConfigDict, Field, PositiveInt + +from inference.core.entities.requests.inference import ( + InstanceSegmentationInferenceRequest, +) +from inference.core.env import ( + HOSTED_INSTANCE_SEGMENTATION_URL, + LOCAL_INFERENCE_API_URL, + WORKFLOWS_REMOTE_API_TARGET, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, +) +from inference.core.managers.base import ModelManager +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.core_steps.common.utils import ( + attach_parents_coordinates_to_batch_of_sv_detections, + attach_prediction_type_info_to_sv_detections_batch, + convert_inference_detections_batch_to_sv_detections, + filter_out_unwanted_classes_from_sv_detections_batch, +) +from inference.core.workflows.execution_engine.constants import INFERENCE_ID_KEY +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, + INFERENCE_ID_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + INTEGER_KIND, + LIST_OF_VALUES_KIND, + ROBOFLOW_MODEL_ID_KIND, + ROBOFLOW_PROJECT_KIND, + STRING_KIND, + FloatZeroToOne, + ImageInputField, + RoboflowModelField, + Selector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) +from inference_sdk import InferenceConfiguration, InferenceHTTPClient + +LONG_DESCRIPTION = """ +Run inference on an instance segmentation model hosted on or uploaded to Roboflow. + +You can query any model that is private to your account, or any public model available +on [Roboflow Universe](https://universe.roboflow.com). + +You will need to set your Roboflow API key in your Inference environment to use this +block. To learn more about setting your Roboflow API key, [refer to the Inference +documentation](https://inference.roboflow.com/quickstart/configure_api_key/). +""" + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Instance Segmentation Model", + "version": "v2", + "short_description": "Predict the shape, size, and location of objects.", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/roboflow_instance_segmentation_model@v2"] + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + class_agnostic_nms: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=False, + description="Value to decide if NMS is to be used in class-agnostic mode.", + examples=[True, "$inputs.class_agnostic_nms"], + ) + class_filter: Union[Optional[List[str]], Selector(kind=[LIST_OF_VALUES_KIND])] = ( + Field( + default=None, + description="List of classes to retrieve from predictions (to define subset of those which was used while model training)", + examples=[["a", "b", "c"], "$inputs.class_filter"], + ) + ) + confidence: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.4, + description="Confidence threshold for predictions", + examples=[0.3, "$inputs.confidence_threshold"], + ) + iou_threshold: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.3, + description="Parameter of NMS, to decide on minimum box intersection over union to merge boxes", + examples=[0.4, "$inputs.iou_threshold"], + ) + max_detections: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + default=300, + description="Maximum number of detections to return", + examples=[300, "$inputs.max_detections"], + ) + max_candidates: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + default=3000, + description="Maximum number of candidates as NMS input to be taken into account.", + examples=[3000, "$inputs.max_candidates"], + ) + mask_decode_mode: Union[ + Literal["accurate", "tradeoff", "fast"], + Selector(kind=[STRING_KIND]), + ] = Field( + default="accurate", + description="Parameter of mask decoding in prediction post-processing.", + examples=["accurate", "$inputs.mask_decode_mode"], + ) + tradeoff_factor: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.0, + description="Post-processing parameter to dictate tradeoff between fast and accurate", + examples=[0.3, "$inputs.tradeoff_factor"], + ) + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Parameter to decide if Active Learning data sampling is disabled for the model", + examples=[True, "$inputs.disable_active_learning"], + ) + active_learning_target_dataset: Union[ + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + ] = Field( + default=None, + description="Target dataset for Active Learning data sampling - see Roboflow Active Learning " + "docs for more information", + examples=["my_project", "$inputs.al_target_project"], + ) + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name=INFERENCE_ID_KEY, kind=[INFERENCE_ID_KIND]), + OutputDefinition( + name="predictions", + kind=[INSTANCE_SEGMENTATION_PREDICTION_KIND], + ), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class RoboflowInstanceSegmentationModelBlockV2(WorkflowBlock): + + def __init__( + self, + model_manager: ModelManager, + api_key: Optional[str], + step_execution_mode: StepExecutionMode, + ): + self._model_manager = model_manager + self._api_key = api_key + self._step_execution_mode = step_execution_mode + + @classmethod + def get_init_parameters(cls) -> List[str]: + return ["model_manager", "api_key", "step_execution_mode"] + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run( + self, + images: Batch[WorkflowImageData], + model_id: str, + class_agnostic_nms: Optional[bool], + class_filter: Optional[List[str]], + confidence: Optional[float], + iou_threshold: Optional[float], + max_detections: Optional[int], + max_candidates: Optional[int], + mask_decode_mode: Literal["accurate", "tradeoff", "fast"], + tradeoff_factor: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + if self._step_execution_mode is StepExecutionMode.LOCAL: + return self.run_locally( + images=images, + model_id=model_id, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + mask_decode_mode=mask_decode_mode, + tradeoff_factor=tradeoff_factor, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + elif self._step_execution_mode is StepExecutionMode.REMOTE: + return self.run_remotely( + images=images, + model_id=model_id, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + mask_decode_mode=mask_decode_mode, + tradeoff_factor=tradeoff_factor, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + else: + raise ValueError( + f"Unknown step execution mode: {self._step_execution_mode}" + ) + + def run_locally( + self, + images: Batch[WorkflowImageData], + model_id: str, + class_agnostic_nms: Optional[bool], + class_filter: Optional[List[str]], + confidence: Optional[float], + iou_threshold: Optional[float], + max_detections: Optional[int], + max_candidates: Optional[int], + mask_decode_mode: Literal["accurate", "tradeoff", "fast"], + tradeoff_factor: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + inference_images = [i.to_inference_format(numpy_preferred=True) for i in images] + request = InstanceSegmentationInferenceRequest( + api_key=self._api_key, + model_id=model_id, + image=inference_images, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + mask_decode_mode=mask_decode_mode, + tradeoff_factor=tradeoff_factor, + source="workflow-execution", + ) + self._model_manager.add_model( + model_id=model_id, + api_key=self._api_key, + ) + predictions = self._model_manager.infer_from_request_sync( + model_id=model_id, request=request + ) + if not isinstance(predictions, list): + predictions = [predictions] + predictions = [ + e.model_dump(by_alias=True, exclude_none=True) for e in predictions + ] + return self._post_process_result( + images=images, + predictions=predictions, + class_filter=class_filter, + ) + + def run_remotely( + self, + images: Batch[WorkflowImageData], + model_id: str, + class_agnostic_nms: Optional[bool], + class_filter: Optional[List[str]], + confidence: Optional[float], + iou_threshold: Optional[float], + max_detections: Optional[int], + max_candidates: Optional[int], + mask_decode_mode: Literal["accurate", "tradeoff", "fast"], + tradeoff_factor: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + api_url = ( + LOCAL_INFERENCE_API_URL + if WORKFLOWS_REMOTE_API_TARGET != "hosted" + else HOSTED_INSTANCE_SEGMENTATION_URL + ) + client = InferenceHTTPClient( + api_url=api_url, + api_key=self._api_key, + ) + if WORKFLOWS_REMOTE_API_TARGET == "hosted": + client.select_api_v0() + client_config = InferenceConfiguration( + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence_threshold=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + mask_decode_mode=mask_decode_mode, + tradeoff_factor=tradeoff_factor, + max_batch_size=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + max_concurrent_requests=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, + source="workflow-execution", + ) + client.configure(inference_configuration=client_config) + inference_images = [i.numpy_image for i in images] + predictions = client.infer( + inference_input=inference_images, + model_id=model_id, + ) + if not isinstance(predictions, list): + predictions = [predictions] + return self._post_process_result( + images=images, + predictions=predictions, + class_filter=class_filter, + ) + + def _post_process_result( + self, + images: Batch[WorkflowImageData], + predictions: List[dict], + class_filter: Optional[List[str]], + ) -> BlockResult: + inference_id = predictions[0].get(INFERENCE_ID_KEY, None) + predictions = convert_inference_detections_batch_to_sv_detections(predictions) + predictions = attach_prediction_type_info_to_sv_detections_batch( + predictions=predictions, + prediction_type="instance-segmentation", + ) + predictions = filter_out_unwanted_classes_from_sv_detections_batch( + predictions=predictions, + classes_to_accept=class_filter, + ) + predictions = attach_parents_coordinates_to_batch_of_sv_detections( + images=images, + predictions=predictions, + ) + return [ + {"inference_id": inference_id, "predictions": prediction} + for prediction in predictions + ] diff --git a/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v1.py b/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v1.py index b9d80bfed..f95d7c04d 100644 --- a/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v1.py @@ -30,6 +30,7 @@ from inference.core.workflows.execution_engine.entities.types import ( BOOLEAN_KIND, FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, INTEGER_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, LIST_OF_VALUES_KIND, @@ -39,9 +40,7 @@ FloatZeroToOne, ImageInputField, RoboflowModelField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -79,27 +78,23 @@ class BlockManifest(WorkflowBlockManifest): "RoboflowKeypointDetectionModel", "KeypointsDetectionModel", ] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - model_id: Union[WorkflowParameterSelector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = ( - RoboflowModelField + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + class_agnostic_nms: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=False, + description="Value to decide if NMS is to be used in class-agnostic mode.", + examples=[True, "$inputs.class_agnostic_nms"], ) - class_agnostic_nms: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = ( + class_filter: Union[Optional[List[str]], Selector(kind=[LIST_OF_VALUES_KIND])] = ( Field( - default=False, - description="Value to decide if NMS is to be used in class-agnostic mode.", - examples=[True, "$inputs.class_agnostic_nms"], + default=None, + description="List of classes to retrieve from predictions (to define subset of those which was used while model training)", + examples=[["a", "b", "c"], "$inputs.class_filter"], ) ) - class_filter: Union[ - Optional[List[str]], WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]) - ] = Field( - default=None, - description="List of classes to retrieve from predictions (to define subset of those which was used while model training)", - examples=[["a", "b", "c"], "$inputs.class_filter"], - ) confidence: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Confidence threshold for predictions", @@ -107,43 +102,37 @@ class BlockManifest(WorkflowBlockManifest): ) iou_threshold: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.3, description="Parameter of NMS, to decide on minimum box intersection over union to merge boxes", examples=[0.4, "$inputs.iou_threshold"], ) - max_detections: Union[ - PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) - ] = Field( + max_detections: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( default=300, description="Maximum number of detections to return", examples=[300, "$inputs.max_detections"], ) - max_candidates: Union[ - PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) - ] = Field( + max_candidates: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( default=3000, description="Maximum number of candidates as NMS input to be taken into account.", examples=[3000, "$inputs.max_candidates"], ) keypoint_confidence: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.0, description="Confidence threshold to predict keypoint as visible.", examples=[0.3, "$inputs.keypoint_confidence"], ) - disable_active_learning: Union[ - bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND]) - ] = Field( + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=True, description="Parameter to decide if Active Learning data sampling is disabled for the model", examples=[True, "$inputs.disable_active_learning"], ) active_learning_target_dataset: Union[ - WorkflowParameterSelector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] ] = Field( default=None, description="Target dataset for Active Learning data sampling - see Roboflow Active Learning " @@ -152,8 +141,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -166,7 +155,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RoboflowKeypointDetectionModelBlockV1(WorkflowBlock): @@ -324,7 +313,7 @@ def run_remotely( source="workflow-execution", ) client.configure(inference_configuration=client_config) - inference_images = [i.numpy_image for i in images] + inference_images = [i.base64_image for i in images] predictions = client.infer( inference_input=inference_images, model_id=model_id, diff --git a/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v2.py b/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v2.py new file mode 100644 index 000000000..974e84443 --- /dev/null +++ b/inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v2.py @@ -0,0 +1,353 @@ +from typing import List, Literal, Optional, Type, Union + +from pydantic import ConfigDict, Field, PositiveInt + +from inference.core.entities.requests.inference import ( + KeypointsDetectionInferenceRequest, +) +from inference.core.env import ( + HOSTED_DETECT_URL, + LOCAL_INFERENCE_API_URL, + WORKFLOWS_REMOTE_API_TARGET, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, +) +from inference.core.managers.base import ModelManager +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.core_steps.common.utils import ( + add_inference_keypoints_to_sv_detections, + attach_parents_coordinates_to_batch_of_sv_detections, + attach_prediction_type_info_to_sv_detections_batch, + convert_inference_detections_batch_to_sv_detections, + filter_out_unwanted_classes_from_sv_detections_batch, +) +from inference.core.workflows.execution_engine.constants import INFERENCE_ID_KEY +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, + INFERENCE_ID_KIND, + INTEGER_KIND, + KEYPOINT_DETECTION_PREDICTION_KIND, + LIST_OF_VALUES_KIND, + ROBOFLOW_MODEL_ID_KIND, + ROBOFLOW_PROJECT_KIND, + FloatZeroToOne, + ImageInputField, + RoboflowModelField, + Selector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) +from inference_sdk import InferenceConfiguration, InferenceHTTPClient + +LONG_DESCRIPTION = """ +Run inference on a keypoint detection model hosted on or uploaded to Roboflow. + +You can query any model that is private to your account, or any public model available +on [Roboflow Universe](https://universe.roboflow.com). + +You will need to set your Roboflow API key in your Inference environment to use this +block. To learn more about setting your Roboflow API key, [refer to the Inference +documentation](https://inference.roboflow.com/quickstart/configure_api_key/). +""" + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Keypoint Detection Model", + "version": "v2", + "short_description": "Predict skeletons on objects.", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/roboflow_keypoint_detection_model@v2"] + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + class_agnostic_nms: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=False, + description="Value to decide if NMS is to be used in class-agnostic mode.", + examples=[True, "$inputs.class_agnostic_nms"], + ) + class_filter: Union[Optional[List[str]], Selector(kind=[LIST_OF_VALUES_KIND])] = ( + Field( + default=None, + description="List of classes to retrieve from predictions (to define subset of those which was used while model training)", + examples=[["a", "b", "c"], "$inputs.class_filter"], + ) + ) + confidence: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.4, + description="Confidence threshold for predictions", + examples=[0.3, "$inputs.confidence_threshold"], + ) + iou_threshold: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.3, + description="Parameter of NMS, to decide on minimum box intersection over union to merge boxes", + examples=[0.4, "$inputs.iou_threshold"], + ) + max_detections: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + default=300, + description="Maximum number of detections to return", + examples=[300, "$inputs.max_detections"], + ) + max_candidates: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + default=3000, + description="Maximum number of candidates as NMS input to be taken into account.", + examples=[3000, "$inputs.max_candidates"], + ) + keypoint_confidence: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.0, + description="Confidence threshold to predict keypoint as visible.", + examples=[0.3, "$inputs.keypoint_confidence"], + ) + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Parameter to decide if Active Learning data sampling is disabled for the model", + examples=[True, "$inputs.disable_active_learning"], + ) + active_learning_target_dataset: Union[ + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + ] = Field( + default=None, + description="Target dataset for Active Learning data sampling - see Roboflow Active Learning " + "docs for more information", + examples=["my_project", "$inputs.al_target_project"], + ) + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name=INFERENCE_ID_KEY, kind=[INFERENCE_ID_KIND]), + OutputDefinition( + name="predictions", kind=[KEYPOINT_DETECTION_PREDICTION_KIND] + ), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class RoboflowKeypointDetectionModelBlockV2(WorkflowBlock): + + def __init__( + self, + model_manager: ModelManager, + api_key: Optional[str], + step_execution_mode: StepExecutionMode, + ): + self._model_manager = model_manager + self._api_key = api_key + self._step_execution_mode = step_execution_mode + + @classmethod + def get_init_parameters(cls) -> List[str]: + return ["model_manager", "api_key", "step_execution_mode"] + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run( + self, + images: Batch[WorkflowImageData], + model_id: str, + class_agnostic_nms: Optional[bool], + class_filter: Optional[List[str]], + confidence: Optional[float], + iou_threshold: Optional[float], + max_detections: Optional[int], + max_candidates: Optional[int], + keypoint_confidence: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + if self._step_execution_mode is StepExecutionMode.LOCAL: + return self.run_locally( + images=images, + model_id=model_id, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + keypoint_confidence=keypoint_confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + elif self._step_execution_mode is StepExecutionMode.REMOTE: + return self.run_remotely( + images=images, + model_id=model_id, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + keypoint_confidence=keypoint_confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + else: + raise ValueError( + f"Unknown step execution mode: {self._step_execution_mode}" + ) + + def run_locally( + self, + images: Batch[WorkflowImageData], + model_id: str, + class_agnostic_nms: Optional[bool], + class_filter: Optional[List[str]], + confidence: Optional[float], + iou_threshold: Optional[float], + max_detections: Optional[int], + max_candidates: Optional[int], + keypoint_confidence: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + inference_images = [i.to_inference_format(numpy_preferred=True) for i in images] + request = KeypointsDetectionInferenceRequest( + api_key=self._api_key, + model_id=model_id, + image=inference_images, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + keypoint_confidence=keypoint_confidence, + source="workflow-execution", + ) + self._model_manager.add_model( + model_id=model_id, + api_key=self._api_key, + ) + predictions = self._model_manager.infer_from_request_sync( + model_id=model_id, request=request + ) + if not isinstance(predictions, list): + predictions = [predictions] + predictions = [ + e.model_dump(by_alias=True, exclude_none=True) for e in predictions + ] + return self._post_process_result( + images=images, + predictions=predictions, + class_filter=class_filter, + ) + + def run_remotely( + self, + images: Batch[Optional[WorkflowImageData]], + model_id: str, + class_agnostic_nms: Optional[bool], + class_filter: Optional[List[str]], + confidence: Optional[float], + iou_threshold: Optional[float], + max_detections: Optional[int], + max_candidates: Optional[int], + keypoint_confidence: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + api_url = ( + LOCAL_INFERENCE_API_URL + if WORKFLOWS_REMOTE_API_TARGET != "hosted" + else HOSTED_DETECT_URL + ) + client = InferenceHTTPClient( + api_url=api_url, + api_key=self._api_key, + ) + if WORKFLOWS_REMOTE_API_TARGET == "hosted": + client.select_api_v0() + client_config = InferenceConfiguration( + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence_threshold=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + keypoint_confidence_threshold=keypoint_confidence, + max_batch_size=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + max_concurrent_requests=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, + source="workflow-execution", + ) + client.configure(inference_configuration=client_config) + inference_images = [i.numpy_image for i in images] + predictions = client.infer( + inference_input=inference_images, + model_id=model_id, + ) + if not isinstance(predictions, list): + predictions = [predictions] + return self._post_process_result( + images=images, + predictions=predictions, + class_filter=class_filter, + ) + + def _post_process_result( + self, + images: Batch[WorkflowImageData], + predictions: List[dict], + class_filter: Optional[List[str]], + ) -> BlockResult: + inference_id = predictions[0].get(INFERENCE_ID_KEY, None) + detections = convert_inference_detections_batch_to_sv_detections(predictions) + for prediction, image_detections in zip(predictions, detections): + add_inference_keypoints_to_sv_detections( + inference_prediction=prediction["predictions"], + detections=image_detections, + ) + detections = attach_prediction_type_info_to_sv_detections_batch( + predictions=detections, + prediction_type="keypoint-detection", + ) + detections = filter_out_unwanted_classes_from_sv_detections_batch( + predictions=detections, + classes_to_accept=class_filter, + ) + detections = attach_parents_coordinates_to_batch_of_sv_detections( + images=images, + predictions=detections, + ) + return [ + {"inference_id": inference_id, "predictions": image_detections} + for image_detections in detections + ] diff --git a/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v1.py b/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v1.py index eca510831..4ff050018 100644 --- a/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v1.py @@ -27,15 +27,14 @@ BOOLEAN_KIND, CLASSIFICATION_PREDICTION_KIND, FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, ROBOFLOW_MODEL_ID_KIND, ROBOFLOW_PROJECT_KIND, STRING_KIND, FloatZeroToOne, ImageInputField, RoboflowModelField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -73,27 +72,23 @@ class BlockManifest(WorkflowBlockManifest): "RoboflowClassificationModel", "ClassificationModel", ] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - model_id: Union[WorkflowParameterSelector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = ( - RoboflowModelField - ) + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField confidence: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Confidence threshold for predictions", examples=[0.3, "$inputs.confidence_threshold"], ) - disable_active_learning: Union[ - bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND]) - ] = Field( + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=True, description="Parameter to decide if Active Learning data sampling is disabled for the model", examples=[True, "$inputs.disable_active_learning"], ) active_learning_target_dataset: Union[ - WorkflowParameterSelector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] ] = Field( default=None, description="Target dataset for Active Learning data sampling - see Roboflow Active Learning " @@ -102,8 +97,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -114,7 +109,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RoboflowClassificationModelBlockV1(WorkflowBlock): @@ -230,7 +225,7 @@ def run_remotely( source="workflow-execution", ) client.configure(inference_configuration=client_config) - non_empty_inference_images = [i.numpy_image for i in images] + non_empty_inference_images = [i.base64_image for i in images] predictions = client.infer( inference_input=non_empty_inference_images, model_id=model_id, @@ -247,7 +242,6 @@ def _post_process_result( images: Batch[WorkflowImageData], predictions: List[dict], ) -> BlockResult: - inference_id = predictions[0].get(INFERENCE_ID_KEY, None) predictions = attach_prediction_type_info( predictions=predictions, prediction_type="classification", @@ -258,6 +252,9 @@ def _post_process_result( image.workflow_root_ancestor_metadata.parent_id ) return [ - {"inference_id": inference_id, "predictions": prediction} + { + "inference_id": prediction.get(INFERENCE_ID_KEY), + "predictions": prediction, + } for prediction in predictions ] diff --git a/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v2.py b/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v2.py new file mode 100644 index 000000000..27022c9ad --- /dev/null +++ b/inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v2.py @@ -0,0 +1,256 @@ +from typing import List, Literal, Optional, Type, Union + +from pydantic import ConfigDict, Field + +from inference.core.entities.requests.inference import ClassificationInferenceRequest +from inference.core.env import ( + HOSTED_CLASSIFICATION_URL, + LOCAL_INFERENCE_API_URL, + WORKFLOWS_REMOTE_API_TARGET, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, +) +from inference.core.managers.base import ModelManager +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.core_steps.common.utils import attach_prediction_type_info +from inference.core.workflows.execution_engine.constants import ( + INFERENCE_ID_KEY, + PARENT_ID_KEY, + ROOT_PARENT_ID_KEY, +) +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + CLASSIFICATION_PREDICTION_KIND, + FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, + INFERENCE_ID_KIND, + ROBOFLOW_MODEL_ID_KIND, + ROBOFLOW_PROJECT_KIND, + FloatZeroToOne, + ImageInputField, + RoboflowModelField, + Selector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) +from inference_sdk import InferenceConfiguration, InferenceHTTPClient + +LONG_DESCRIPTION = """ +Run inference on a multi-class classification model hosted on or uploaded to Roboflow. + +You can query any model that is private to your account, or any public model available +on [Roboflow Universe](https://universe.roboflow.com). + +You will need to set your Roboflow API key in your Inference environment to use this +block. To learn more about setting your Roboflow API key, [refer to the Inference +documentation](https://inference.roboflow.com/quickstart/configure_api_key/). +""" + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Single-Label Classification Model", + "version": "v2", + "short_description": "Apply a single tag to an image.", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/roboflow_classification_model@v2"] + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + confidence: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.4, + description="Confidence threshold for predictions", + examples=[0.3, "$inputs.confidence_threshold"], + ) + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Parameter to decide if Active Learning data sampling is disabled for the model", + examples=[True, "$inputs.disable_active_learning"], + ) + active_learning_target_dataset: Union[ + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + ] = Field( + default=None, + description="Target dataset for Active Learning data sampling - see Roboflow Active Learning " + "docs for more information", + examples=["my_project", "$inputs.al_target_project"], + ) + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="predictions", kind=[CLASSIFICATION_PREDICTION_KIND]), + OutputDefinition(name=INFERENCE_ID_KEY, kind=[INFERENCE_ID_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class RoboflowClassificationModelBlockV2(WorkflowBlock): + + def __init__( + self, + model_manager: ModelManager, + api_key: Optional[str], + step_execution_mode: StepExecutionMode, + ): + self._model_manager = model_manager + self._api_key = api_key + self._step_execution_mode = step_execution_mode + + @classmethod + def get_init_parameters(cls) -> List[str]: + return ["model_manager", "api_key", "step_execution_mode"] + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run( + self, + images: Batch[WorkflowImageData], + model_id: str, + confidence: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + if self._step_execution_mode is StepExecutionMode.LOCAL: + return self.run_locally( + images=images, + model_id=model_id, + confidence=confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + elif self._step_execution_mode is StepExecutionMode.REMOTE: + return self.run_remotely( + images=images, + model_id=model_id, + confidence=confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + else: + raise ValueError( + f"Unknown step execution mode: {self._step_execution_mode}" + ) + + def run_locally( + self, + images: Batch[WorkflowImageData], + model_id: str, + confidence: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + inference_images = [i.to_inference_format(numpy_preferred=True) for i in images] + request = ClassificationInferenceRequest( + api_key=self._api_key, + model_id=model_id, + image=inference_images, + confidence=confidence, + disable_active_learning=disable_active_learning, + source="workflow-execution", + active_learning_target_dataset=active_learning_target_dataset, + ) + self._model_manager.add_model( + model_id=model_id, + api_key=self._api_key, + ) + predictions = self._model_manager.infer_from_request_sync( + model_id=model_id, request=request + ) + if isinstance(predictions, list): + predictions = [ + e.model_dump(by_alias=True, exclude_none=True) for e in predictions + ] + else: + predictions = [predictions.model_dump(by_alias=True, exclude_none=True)] + return self._post_process_result( + predictions=predictions, + images=images, + ) + + def run_remotely( + self, + images: Batch[Optional[WorkflowImageData]], + model_id: str, + confidence: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + api_url = ( + LOCAL_INFERENCE_API_URL + if WORKFLOWS_REMOTE_API_TARGET != "hosted" + else HOSTED_CLASSIFICATION_URL + ) + client = InferenceHTTPClient( + api_url=api_url, + api_key=self._api_key, + ) + if WORKFLOWS_REMOTE_API_TARGET == "hosted": + client.select_api_v0() + client_config = InferenceConfiguration( + confidence_threshold=confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + max_batch_size=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + max_concurrent_requests=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, + source="workflow-execution", + ) + client.configure(inference_configuration=client_config) + non_empty_inference_images = [i.numpy_image for i in images] + predictions = client.infer( + inference_input=non_empty_inference_images, + model_id=model_id, + ) + if not isinstance(predictions, list): + predictions = [predictions] + return self._post_process_result( + predictions=predictions, + images=images, + ) + + def _post_process_result( + self, + images: Batch[WorkflowImageData], + predictions: List[dict], + ) -> BlockResult: + predictions = attach_prediction_type_info( + predictions=predictions, + prediction_type="classification", + ) + for prediction, image in zip(predictions, images): + prediction[PARENT_ID_KEY] = image.parent_metadata.parent_id + prediction[ROOT_PARENT_ID_KEY] = ( + image.workflow_root_ancestor_metadata.parent_id + ) + return [ + { + "inference_id": prediction.get(INFERENCE_ID_KEY), + "predictions": prediction, + } + for prediction in predictions + ] diff --git a/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v1.py b/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v1.py index 78b41b32b..41de72787 100644 --- a/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v1.py @@ -27,15 +27,14 @@ BOOLEAN_KIND, CLASSIFICATION_PREDICTION_KIND, FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, ROBOFLOW_MODEL_ID_KIND, ROBOFLOW_PROJECT_KIND, STRING_KIND, FloatZeroToOne, ImageInputField, RoboflowModelField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -73,27 +72,23 @@ class BlockManifest(WorkflowBlockManifest): "RoboflowMultiLabelClassificationModel", "MultiLabelClassificationModel", ] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - model_id: Union[WorkflowParameterSelector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = ( - RoboflowModelField - ) + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField confidence: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Confidence threshold for predictions", examples=[0.3, "$inputs.confidence_threshold"], ) - disable_active_learning: Union[ - bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND]) - ] = Field( + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=True, description="Parameter to decide if Active Learning data sampling is disabled for the model", examples=[True, "$inputs.disable_active_learning"], ) active_learning_target_dataset: Union[ - WorkflowParameterSelector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] ] = Field( default=None, description="Target dataset for Active Learning data sampling - see Roboflow Active Learning " @@ -102,8 +97,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -114,7 +109,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RoboflowMultiLabelClassificationModelBlockV1(WorkflowBlock): @@ -230,7 +225,7 @@ def run_remotely( source="workflow-execution", ) client.configure(inference_configuration=client_config) - non_empty_inference_images = [i.numpy_image for i in images] + non_empty_inference_images = [i.base64_image for i in images] predictions = client.infer( inference_input=non_empty_inference_images, model_id=model_id, @@ -244,7 +239,6 @@ def _post_process_result( images: Batch[WorkflowImageData], predictions: List[dict], ) -> List[dict]: - inference_id = predictions[0].get(INFERENCE_ID_KEY, None) predictions = attach_prediction_type_info( predictions=predictions, prediction_type="classification", @@ -255,6 +249,9 @@ def _post_process_result( image.workflow_root_ancestor_metadata.parent_id ) return [ - {"inference_id": inference_id, "predictions": prediction} + { + "inference_id": prediction.get(INFERENCE_ID_KEY), + "predictions": prediction, + } for prediction in predictions ] diff --git a/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v2.py b/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v2.py new file mode 100644 index 000000000..500441ac3 --- /dev/null +++ b/inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v2.py @@ -0,0 +1,254 @@ +from typing import List, Literal, Optional, Type, Union + +from pydantic import ConfigDict, Field + +from inference.core.entities.requests.inference import ClassificationInferenceRequest +from inference.core.env import ( + HOSTED_CLASSIFICATION_URL, + LOCAL_INFERENCE_API_URL, + WORKFLOWS_REMOTE_API_TARGET, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, +) +from inference.core.managers.base import ModelManager +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.core_steps.common.utils import attach_prediction_type_info +from inference.core.workflows.execution_engine.constants import ( + INFERENCE_ID_KEY, + PARENT_ID_KEY, + ROOT_PARENT_ID_KEY, +) +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + CLASSIFICATION_PREDICTION_KIND, + FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, + INFERENCE_ID_KIND, + ROBOFLOW_MODEL_ID_KIND, + ROBOFLOW_PROJECT_KIND, + FloatZeroToOne, + ImageInputField, + RoboflowModelField, + Selector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) +from inference_sdk import InferenceConfiguration, InferenceHTTPClient + +LONG_DESCRIPTION = """ +Run inference on a multi-label classification model hosted on or uploaded to Roboflow. + +You can query any model that is private to your account, or any public model available +on [Roboflow Universe](https://universe.roboflow.com). + +You will need to set your Roboflow API key in your Inference environment to use this +block. To learn more about setting your Roboflow API key, [refer to the Inference +documentation](https://inference.roboflow.com/quickstart/configure_api_key/). +""" + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Multi-Label Classification Model", + "version": "v2", + "short_description": "Apply multiple tags to an image.", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/roboflow_multi_label_classification_model@v2"] + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + confidence: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.4, + description="Confidence threshold for predictions", + examples=[0.3, "$inputs.confidence_threshold"], + ) + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Parameter to decide if Active Learning data sampling is disabled for the model", + examples=[True, "$inputs.disable_active_learning"], + ) + active_learning_target_dataset: Union[ + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + ] = Field( + default=None, + description="Target dataset for Active Learning data sampling - see Roboflow Active Learning " + "docs for more information", + examples=["my_project", "$inputs.al_target_project"], + ) + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="predictions", kind=[CLASSIFICATION_PREDICTION_KIND]), + OutputDefinition(name=INFERENCE_ID_KEY, kind=[INFERENCE_ID_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class RoboflowMultiLabelClassificationModelBlockV2(WorkflowBlock): + + def __init__( + self, + model_manager: ModelManager, + api_key: Optional[str], + step_execution_mode: StepExecutionMode, + ): + self._model_manager = model_manager + self._api_key = api_key + self._step_execution_mode = step_execution_mode + + @classmethod + def get_init_parameters(cls) -> List[str]: + return ["model_manager", "api_key", "step_execution_mode"] + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run( + self, + images: Batch[WorkflowImageData], + model_id: str, + confidence: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + if self._step_execution_mode is StepExecutionMode.LOCAL: + return self.run_locally( + images=images, + model_id=model_id, + confidence=confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + elif self._step_execution_mode is StepExecutionMode.REMOTE: + return self.run_remotely( + images=images, + model_id=model_id, + confidence=confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + else: + raise ValueError( + f"Unknown step execution mode: {self._step_execution_mode}" + ) + + def run_locally( + self, + images: Batch[WorkflowImageData], + model_id: str, + confidence: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + inference_images = [i.to_inference_format(numpy_preferred=True) for i in images] + request = ClassificationInferenceRequest( + api_key=self._api_key, + model_id=model_id, + image=inference_images, + confidence=confidence, + disable_active_learning=disable_active_learning, + source="workflow-execution", + active_learning_target_dataset=active_learning_target_dataset, + ) + self._model_manager.add_model( + model_id=model_id, + api_key=self._api_key, + ) + predictions = self._model_manager.infer_from_request_sync( + model_id=model_id, request=request + ) + if isinstance(predictions, list): + predictions = [ + e.dict(by_alias=True, exclude_none=True) for e in predictions + ] + else: + predictions = [predictions.dict(by_alias=True, exclude_none=True)] + return self._post_process_result( + predictions=predictions, + images=images, + ) + + def run_remotely( + self, + images: Batch[Optional[WorkflowImageData]], + model_id: str, + confidence: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + api_url = ( + LOCAL_INFERENCE_API_URL + if WORKFLOWS_REMOTE_API_TARGET != "hosted" + else HOSTED_CLASSIFICATION_URL + ) + client = InferenceHTTPClient( + api_url=api_url, + api_key=self._api_key, + ) + if WORKFLOWS_REMOTE_API_TARGET == "hosted": + client.select_api_v0() + client_config = InferenceConfiguration( + confidence_threshold=confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + max_batch_size=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + max_concurrent_requests=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, + source="workflow-execution", + ) + client.configure(inference_configuration=client_config) + non_empty_inference_images = [i.numpy_image for i in images] + predictions = client.infer( + inference_input=non_empty_inference_images, + model_id=model_id, + ) + if not isinstance(predictions, list): + predictions = [predictions] + return self._post_process_result(images=images, predictions=predictions) + + def _post_process_result( + self, + images: Batch[WorkflowImageData], + predictions: List[dict], + ) -> List[dict]: + inference_id = predictions[0].get(INFERENCE_ID_KEY, None) + predictions = attach_prediction_type_info( + predictions=predictions, + prediction_type="classification", + ) + for prediction, image in zip(predictions, images): + prediction[PARENT_ID_KEY] = image.parent_metadata.parent_id + prediction[ROOT_PARENT_ID_KEY] = ( + image.workflow_root_ancestor_metadata.parent_id + ) + return [ + { + "inference_id": prediction.get(INFERENCE_ID_KEY), + "predictions": prediction, + } + for prediction in predictions + ] diff --git a/inference/core/workflows/core_steps/models/roboflow/object_detection/v1.py b/inference/core/workflows/core_steps/models/roboflow/object_detection/v1.py index ab8b84f26..486752a63 100644 --- a/inference/core/workflows/core_steps/models/roboflow/object_detection/v1.py +++ b/inference/core/workflows/core_steps/models/roboflow/object_detection/v1.py @@ -27,6 +27,7 @@ from inference.core.workflows.execution_engine.entities.types import ( BOOLEAN_KIND, FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, INTEGER_KIND, LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, @@ -36,9 +37,7 @@ FloatZeroToOne, ImageInputField, RoboflowModelField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -76,27 +75,23 @@ class BlockManifest(WorkflowBlockManifest): "RoboflowObjectDetectionModel", "ObjectDetectionModel", ] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - model_id: Union[WorkflowParameterSelector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = ( - RoboflowModelField - ) - class_agnostic_nms: Union[ - Optional[bool], WorkflowParameterSelector(kind=[BOOLEAN_KIND]) - ] = Field( + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + class_agnostic_nms: Union[Optional[bool], Selector(kind=[BOOLEAN_KIND])] = Field( default=False, description="Value to decide if NMS is to be used in class-agnostic mode.", examples=[True, "$inputs.class_agnostic_nms"], ) - class_filter: Union[ - Optional[List[str]], WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]) - ] = Field( - default=None, - description="List of classes to retrieve from predictions (to define subset of those which was used while model training)", - examples=[["a", "b", "c"], "$inputs.class_filter"], + class_filter: Union[Optional[List[str]], Selector(kind=[LIST_OF_VALUES_KIND])] = ( + Field( + default=None, + description="List of classes to retrieve from predictions (to define subset of those which was used while model training)", + examples=[["a", "b", "c"], "$inputs.class_filter"], + ) ) confidence: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.4, description="Confidence threshold for predictions", @@ -104,35 +99,29 @@ class BlockManifest(WorkflowBlockManifest): ) iou_threshold: Union[ FloatZeroToOne, - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.3, description="Parameter of NMS, to decide on minimum box intersection over union to merge boxes", examples=[0.4, "$inputs.iou_threshold"], ) - max_detections: Union[ - PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) - ] = Field( + max_detections: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( default=300, description="Maximum number of detections to return", examples=[300, "$inputs.max_detections"], ) - max_candidates: Union[ - PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) - ] = Field( + max_candidates: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( default=3000, description="Maximum number of candidates as NMS input to be taken into account.", examples=[3000, "$inputs.max_candidates"], ) - disable_active_learning: Union[ - bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND]) - ] = Field( + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=True, description="Parameter to decide if Active Learning data sampling is disabled for the model", examples=[True, "$inputs.disable_active_learning"], ) active_learning_target_dataset: Union[ - WorkflowParameterSelector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] ] = Field( default=None, description="Target dataset for Active Learning data sampling - see Roboflow Active Learning " @@ -141,8 +130,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -155,7 +144,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RoboflowObjectDetectionModelBlockV1(WorkflowBlock): @@ -306,7 +295,7 @@ def run_remotely( source="workflow-execution", ) client.configure(inference_configuration=client_config) - non_empty_inference_images = [i.numpy_image for i in images] + non_empty_inference_images = [i.base64_image for i in images] predictions = client.infer( inference_input=non_empty_inference_images, model_id=model_id, diff --git a/inference/core/workflows/core_steps/models/roboflow/object_detection/v2.py b/inference/core/workflows/core_steps/models/roboflow/object_detection/v2.py new file mode 100644 index 000000000..32889dc1c --- /dev/null +++ b/inference/core/workflows/core_steps/models/roboflow/object_detection/v2.py @@ -0,0 +1,330 @@ +from typing import List, Literal, Optional, Type, Union + +from pydantic import ConfigDict, Field, PositiveInt + +from inference.core.entities.requests.inference import ObjectDetectionInferenceRequest +from inference.core.env import ( + HOSTED_DETECT_URL, + LOCAL_INFERENCE_API_URL, + WORKFLOWS_REMOTE_API_TARGET, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, +) +from inference.core.managers.base import ModelManager +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.core_steps.common.utils import ( + attach_parents_coordinates_to_batch_of_sv_detections, + attach_prediction_type_info_to_sv_detections_batch, + convert_inference_detections_batch_to_sv_detections, + filter_out_unwanted_classes_from_sv_detections_batch, +) +from inference.core.workflows.execution_engine.constants import INFERENCE_ID_KEY +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, + INFERENCE_ID_KIND, + INTEGER_KIND, + LIST_OF_VALUES_KIND, + OBJECT_DETECTION_PREDICTION_KIND, + ROBOFLOW_MODEL_ID_KIND, + ROBOFLOW_PROJECT_KIND, + FloatZeroToOne, + ImageInputField, + RoboflowModelField, + Selector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) +from inference_sdk import InferenceConfiguration, InferenceHTTPClient + +LONG_DESCRIPTION = """ +Run inference on a object-detection model hosted on or uploaded to Roboflow. + +You can query any model that is private to your account, or any public model available +on [Roboflow Universe](https://universe.roboflow.com). + +You will need to set your Roboflow API key in your Inference environment to use this +block. To learn more about setting your Roboflow API key, [refer to the Inference +documentation](https://inference.roboflow.com/quickstart/configure_api_key/). +""" + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Object Detection Model", + "version": "v2", + "short_description": "Predict the location of objects with bounding boxes.", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/roboflow_object_detection_model@v2"] + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + class_agnostic_nms: Union[Optional[bool], Selector(kind=[BOOLEAN_KIND])] = Field( + default=False, + description="Value to decide if NMS is to be used in class-agnostic mode.", + examples=[True, "$inputs.class_agnostic_nms"], + ) + class_filter: Union[Optional[List[str]], Selector(kind=[LIST_OF_VALUES_KIND])] = ( + Field( + default=None, + description="List of classes to retrieve from predictions (to define subset of those which was used while model training)", + examples=[["a", "b", "c"], "$inputs.class_filter"], + ) + ) + confidence: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.4, + description="Confidence threshold for predictions", + examples=[0.3, "$inputs.confidence_threshold"], + ) + iou_threshold: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.3, + description="Parameter of NMS, to decide on minimum box intersection over union to merge boxes", + examples=[0.4, "$inputs.iou_threshold"], + ) + max_detections: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + default=300, + description="Maximum number of detections to return", + examples=[300, "$inputs.max_detections"], + ) + max_candidates: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + default=3000, + description="Maximum number of candidates as NMS input to be taken into account.", + examples=[3000, "$inputs.max_candidates"], + ) + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Parameter to decide if Active Learning data sampling is disabled for the model", + examples=[True, "$inputs.disable_active_learning"], + ) + active_learning_target_dataset: Union[ + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + ] = Field( + default=None, + description="Target dataset for Active Learning data sampling - see Roboflow Active Learning " + "docs for more information", + examples=["my_project", "$inputs.al_target_project"], + ) + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="inference_id", kind=[INFERENCE_ID_KIND]), + OutputDefinition( + name="predictions", kind=[OBJECT_DETECTION_PREDICTION_KIND] + ), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class RoboflowObjectDetectionModelBlockV2(WorkflowBlock): + + def __init__( + self, + model_manager: ModelManager, + api_key: Optional[str], + step_execution_mode: StepExecutionMode, + ): + self._model_manager = model_manager + self._api_key = api_key + self._step_execution_mode = step_execution_mode + + @classmethod + def get_init_parameters(cls) -> List[str]: + return ["model_manager", "api_key", "step_execution_mode"] + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run( + self, + images: Batch[WorkflowImageData], + model_id: str, + class_agnostic_nms: Optional[bool], + class_filter: Optional[List[str]], + confidence: Optional[float], + iou_threshold: Optional[float], + max_detections: Optional[int], + max_candidates: Optional[int], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + if self._step_execution_mode is StepExecutionMode.LOCAL: + return self.run_locally( + images=images, + model_id=model_id, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + elif self._step_execution_mode is StepExecutionMode.REMOTE: + return self.run_remotely( + images=images, + model_id=model_id, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + else: + raise ValueError( + f"Unknown step execution mode: {self._step_execution_mode}" + ) + + def run_locally( + self, + images: Batch[WorkflowImageData], + model_id: str, + class_agnostic_nms: Optional[bool], + class_filter: Optional[List[str]], + confidence: Optional[float], + iou_threshold: Optional[float], + max_detections: Optional[int], + max_candidates: Optional[int], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + inference_images = [i.to_inference_format(numpy_preferred=True) for i in images] + request = ObjectDetectionInferenceRequest( + api_key=self._api_key, + model_id=model_id, + image=inference_images, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + source="workflow-execution", + ) + self._model_manager.add_model( + model_id=model_id, + api_key=self._api_key, + ) + predictions = self._model_manager.infer_from_request_sync( + model_id=model_id, request=request + ) + if not isinstance(predictions, list): + predictions = [predictions] + predictions = [ + e.model_dump(by_alias=True, exclude_none=True) for e in predictions + ] + return self._post_process_result( + images=images, + predictions=predictions, + class_filter=class_filter, + ) + + def run_remotely( + self, + images: Batch[WorkflowImageData], + model_id: str, + class_agnostic_nms: Optional[bool], + class_filter: Optional[List[str]], + confidence: Optional[float], + iou_threshold: Optional[float], + max_detections: Optional[int], + max_candidates: Optional[int], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + api_url = ( + LOCAL_INFERENCE_API_URL + if WORKFLOWS_REMOTE_API_TARGET != "hosted" + else HOSTED_DETECT_URL + ) + client = InferenceHTTPClient( + api_url=api_url, + api_key=self._api_key, + ) + if WORKFLOWS_REMOTE_API_TARGET == "hosted": + client.select_api_v0() + client_config = InferenceConfiguration( + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + class_agnostic_nms=class_agnostic_nms, + class_filter=class_filter, + confidence_threshold=confidence, + iou_threshold=iou_threshold, + max_detections=max_detections, + max_candidates=max_candidates, + max_batch_size=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + max_concurrent_requests=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, + source="workflow-execution", + ) + client.configure(inference_configuration=client_config) + non_empty_inference_images = [i.numpy_image for i in images] + predictions = client.infer( + inference_input=non_empty_inference_images, + model_id=model_id, + ) + if not isinstance(predictions, list): + predictions = [predictions] + return self._post_process_result( + images=images, + predictions=predictions, + class_filter=class_filter, + ) + + def _post_process_result( + self, + images: Batch[WorkflowImageData], + predictions: List[dict], + class_filter: Optional[List[str]], + ) -> BlockResult: + inference_id = predictions[0].get(INFERENCE_ID_KEY, None) + predictions = convert_inference_detections_batch_to_sv_detections(predictions) + predictions = attach_prediction_type_info_to_sv_detections_batch( + predictions=predictions, + prediction_type="object-detection", + ) + predictions = filter_out_unwanted_classes_from_sv_detections_batch( + predictions=predictions, + classes_to_accept=class_filter, + ) + predictions = attach_parents_coordinates_to_batch_of_sv_detections( + images=images, + predictions=predictions, + ) + return [ + {"inference_id": inference_id, "predictions": prediction} + for prediction in predictions + ] diff --git a/inference/core/workflows/core_steps/models/third_party/barcode_detection/v1.py b/inference/core/workflows/core_steps/models/third_party/barcode_detection/v1.py index 052406c6b..e48c1f0a1 100644 --- a/inference/core/workflows/core_steps/models/third_party/barcode_detection/v1.py +++ b/inference/core/workflows/core_steps/models/third_party/barcode_detection/v1.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional, Type, Union +from typing import List, Literal, Optional, Type from uuid import uuid4 import numpy as np @@ -23,9 +23,9 @@ ) from inference.core.workflows.execution_engine.entities.types import ( BAR_CODE_DETECTION_KIND, + IMAGE_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -56,11 +56,11 @@ class BlockManifest(WorkflowBlockManifest): type: Literal[ "roboflow_core/barcode_detector@v1", "BarcodeDetector", "BarcodeDetection" ] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField + images: Selector(kind=[IMAGE_KIND]) = ImageInputField @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -68,7 +68,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class BarcodeDetectorBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/models/third_party/qr_code_detection/v1.py b/inference/core/workflows/core_steps/models/third_party/qr_code_detection/v1.py index ab4b3dfd5..ff9ed2dd6 100644 --- a/inference/core/workflows/core_steps/models/third_party/qr_code_detection/v1.py +++ b/inference/core/workflows/core_steps/models/third_party/qr_code_detection/v1.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional, Type, Union +from typing import List, Literal, Optional, Type from uuid import uuid4 import cv2 @@ -22,10 +22,10 @@ WorkflowImageData, ) from inference.core.workflows.execution_engine.entities.types import ( + IMAGE_KIND, QR_CODE_DETECTION_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -56,11 +56,11 @@ class BlockManifest(WorkflowBlockManifest): type: Literal[ "roboflow_core/qr_code_detector@v1", "QRCodeDetector", "QRCodeDetection" ] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField + images: Selector(kind=[IMAGE_KIND]) = ImageInputField @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -70,7 +70,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class QRCodeDetectorBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/sinks/email_notification/v1.py b/inference/core/workflows/core_steps/sinks/email_notification/v1.py index 6c5f7def5..7fa250823 100644 --- a/inference/core/workflows/core_steps/sinks/email_notification/v1.py +++ b/inference/core/workflows/core_steps/sinks/email_notification/v1.py @@ -29,8 +29,7 @@ INTEGER_KIND, LIST_OF_VALUES_KIND, STRING_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -60,7 +59,7 @@ message using dynamic parameters: ``` -message = "This is example notification. Predicted classes: {{ $parameters.predicted_classes }}" +message = "This is example notification. Predicted classes: \{\{ $parameters.predicted_classes \}\}" ``` Message parameters are delivered by Workflows Execution Engine by setting proper data selectors in @@ -175,17 +174,17 @@ class BlockManifest(WorkflowBlockManifest): message: str = Field( description="Content of the message to be send", examples=[ - "During last 5 minutes detected {{ $parameters.num_instances }} instances" + "During last 5 minutes detected \{\{ $parameters.num_instances \}\} instances" ], ) - sender_email: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( + sender_email: Union[str, Selector(kind=[STRING_KIND])] = Field( description="E-mail to be used to send the message", examples=["sender@gmail.com"], ) receiver_email: Union[ str, List[str], - WorkflowParameterSelector(kind=[STRING_KIND, LIST_OF_VALUES_KIND]), + Selector(kind=[STRING_KIND, LIST_OF_VALUES_KIND]), ] = Field( description="Destination e-mail address", examples=["receiver@gmail.com"], @@ -194,7 +193,7 @@ class BlockManifest(WorkflowBlockManifest): Union[ str, List[str], - WorkflowParameterSelector(kind=[STRING_KIND, LIST_OF_VALUES_KIND]), + Selector(kind=[STRING_KIND, LIST_OF_VALUES_KIND]), ] ] = Field( default=None, @@ -205,7 +204,7 @@ class BlockManifest(WorkflowBlockManifest): Union[ str, List[str], - WorkflowParameterSelector(kind=[STRING_KIND, LIST_OF_VALUES_KIND]), + Selector(kind=[STRING_KIND, LIST_OF_VALUES_KIND]), ] ] = Field( default=None, @@ -214,7 +213,7 @@ class BlockManifest(WorkflowBlockManifest): ) message_parameters: Dict[ str, - Union[WorkflowParameterSelector(), StepOutputSelector(), str, int, float, bool], + Union[Selector(), Selector(), str, int, float, bool], ] = Field( description="References data to be used to construct each and every column", examples=[ @@ -236,21 +235,19 @@ class BlockManifest(WorkflowBlockManifest): ], default_factory=dict, ) - attachments: Dict[str, StepOutputSelector(kind=[STRING_KIND, BYTES_KIND])] = Field( + attachments: Dict[str, Selector(kind=[STRING_KIND, BYTES_KIND])] = Field( description="Attachments", default_factory=dict, examples=[{"report.cvs": "$steps.csv_formatter.csv_content"}], ) - smtp_server: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( + smtp_server: Union[str, Selector(kind=[STRING_KIND])] = Field( description="Custom SMTP server to be used", examples=["$inputs.smtp_server", "smtp.google.com"], ) - sender_email_password: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = ( - Field( - description="Sender e-mail password be used when authenticating to SMTP server", - private=True, - examples=["$inputs.email_password"], - ) + sender_email_password: Union[str, Selector(kind=[STRING_KIND])] = Field( + description="Sender e-mail password be used when authenticating to SMTP server", + private=True, + examples=["$inputs.email_password"], ) smtp_port: int = Field( default=465, @@ -260,30 +257,26 @@ class BlockManifest(WorkflowBlockManifest): "always_visible": True, }, ) - fire_and_forget: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = ( - Field( - default=True, - description="Boolean flag dictating if sink is supposed to be executed in the background, " - "not waiting on status of registration before end of workflow run. Use `True` if best-effort " - "registration is needed, use `False` while debugging and if error handling is needed", - examples=["$inputs.fire_and_forget", False], - ) + fire_and_forget: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Boolean flag dictating if sink is supposed to be executed in the background, " + "not waiting on status of registration before end of workflow run. Use `True` if best-effort " + "registration is needed, use `False` while debugging and if error handling is needed", + examples=["$inputs.fire_and_forget", False], ) - disable_sink: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( + disable_sink: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=False, description="boolean flag that can be also reference to input - to arbitrarily disable " "data collection for specific request", examples=[False, "$inputs.disable_email_notifications"], ) - cooldown_seconds: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = ( - Field( - default=5, - description="Number of seconds to wait until follow-up notification can be sent", - examples=["$inputs.cooldown_seconds", 3], - json_schema_extra={ - "always_visible": True, - }, - ) + cooldown_seconds: Union[int, Selector(kind=[INTEGER_KIND])] = Field( + default=5, + description="Number of seconds to wait until follow-up notification can be sent", + examples=["$inputs.cooldown_seconds", 3], + json_schema_extra={ + "always_visible": True, + }, ) @field_validator("receiver_email") @@ -305,7 +298,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class EmailNotificationBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/sinks/local_file/v1.py b/inference/core/workflows/core_steps/sinks/local_file/v1.py index 11b638c4d..f7fcfc11f 100644 --- a/inference/core/workflows/core_steps/sinks/local_file/v1.py +++ b/inference/core/workflows/core_steps/sinks/local_file/v1.py @@ -11,8 +11,7 @@ from inference.core.workflows.execution_engine.entities.types import ( BOOLEAN_KIND, STRING_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -77,7 +76,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/local_file_sink@v1"] - content: StepOutputSelector(kind=[STRING_KIND]) = Field( + content: Selector(kind=[STRING_KIND]) = Field( description="Content of the file to save", examples=["$steps.csv_formatter.csv_content"], ) @@ -102,11 +101,11 @@ class BlockManifest(WorkflowBlockManifest): } }, ) - target_directory: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + target_directory: Union[Selector(kind=[STRING_KIND]), str] = Field( description="Target directory", examples=["some/location"], ) - file_name_prefix: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + file_name_prefix: Union[Selector(kind=[STRING_KIND]), str] = Field( default="workflow_output", description="File name prefix", examples=["my_file"], @@ -114,20 +113,18 @@ class BlockManifest(WorkflowBlockManifest): "always_visible": True, }, ) - max_entries_per_file: Union[int, WorkflowParameterSelector(kind=[STRING_KIND])] = ( - Field( - default=1024, - description="Defines how many datapoints can be appended to a single file", - examples=[1024], - json_schema_extra={ - "relevant_for": { - "output_mode": { - "values": ["append_log"], - "required": True, - }, - } - }, - ) + max_entries_per_file: Union[int, Selector(kind=[STRING_KIND])] = Field( + default=1024, + description="Defines how many datapoints can be appended to a single file", + examples=[1024], + json_schema_extra={ + "relevant_for": { + "output_mode": { + "values": ["append_log"], + "required": True, + }, + } + }, ) @field_validator("max_entries_per_file") @@ -146,7 +143,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class LocalFileSinkBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/sinks/roboflow/custom_metadata/v1.py b/inference/core/workflows/core_steps/sinks/roboflow/custom_metadata/v1.py index ffc83eaa3..8346fd297 100644 --- a/inference/core/workflows/core_steps/sinks/roboflow/custom_metadata/v1.py +++ b/inference/core/workflows/core_steps/sinks/roboflow/custom_metadata/v1.py @@ -20,8 +20,7 @@ KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -55,7 +54,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/roboflow_custom_metadata@v1", "RoboflowCustomMetadata"] - predictions: StepOutputSelector( + predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -68,8 +67,8 @@ class BlockManifest(WorkflowBlockManifest): ) field_value: Union[ str, - WorkflowParameterSelector(kind=[STRING_KIND]), - StepOutputSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( description="This is the name of the metadata field you are creating", examples=["toronto", "pass", "fail"], @@ -78,14 +77,12 @@ class BlockManifest(WorkflowBlockManifest): description="Name of the field to be updated in Roboflow Customer Metadata", examples=["The name of the value of the field"], ) - fire_and_forget: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = ( - Field( - default=True, - description="Boolean flag dictating if sink is supposed to be executed in the background, " - "not waiting on status of registration before end of workflow run. Use `True` if best-effort " - "registration is needed, use `False` while debugging and if error handling is needed", - examples=[True], - ) + fire_and_forget: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Boolean flag dictating if sink is supposed to be executed in the background, " + "not waiting on status of registration before end of workflow run. Use `True` if best-effort " + "registration is needed, use `False` while debugging and if error handling is needed", + examples=[True], ) @classmethod @@ -97,7 +94,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RoboflowCustomMetadataBlockV1(WorkflowBlock): @@ -127,7 +124,7 @@ def run( fire_and_forget: bool, field_name: str, field_value: str, - predictions: sv.Detections, + predictions: Union[sv.Detections, dict], ) -> BlockResult: if self._api_key is None: raise ValueError( @@ -136,7 +133,11 @@ def run( "https://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key to learn how to " "retrieve one." ) - inference_ids: List[str] = predictions.data.get(INFERENCE_ID_KEY, []) + inference_ids: List[str] = [] + if isinstance(predictions, sv.Detections): + inference_ids = predictions.data.get(INFERENCE_ID_KEY, []) + elif INFERENCE_ID_KEY in predictions: + inference_ids: List[str] = [predictions[INFERENCE_ID_KEY]] if len(inference_ids) == 0: return { "error_status": True, diff --git a/inference/core/workflows/core_steps/sinks/roboflow/dataset_upload/v1.py b/inference/core/workflows/core_steps/sinks/roboflow/dataset_upload/v1.py index a1ee3e75b..2080a21f9 100644 --- a/inference/core/workflows/core_steps/sinks/roboflow/dataset_upload/v1.py +++ b/inference/core/workflows/core_steps/sinks/roboflow/dataset_upload/v1.py @@ -58,16 +58,14 @@ from inference.core.workflows.execution_engine.entities.types import ( BOOLEAN_KIND, CLASSIFICATION_PREDICTION_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, ROBOFLOW_PROJECT_KIND, STRING_KIND, ImageInputField, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -104,9 +102,9 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/roboflow_dataset_upload@v1", "RoboflowDatasetUpload"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField + images: Selector(kind=[IMAGE_KIND]) = ImageInputField predictions: Optional[ - StepOutputSelector( + Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -119,9 +117,7 @@ class BlockManifest(WorkflowBlockManifest): description="Reference q detection-like predictions", examples=["$steps.object_detection_model.predictions"], ) - target_project: Union[ - WorkflowParameterSelector(kind=[ROBOFLOW_PROJECT_KIND]), str - ] = Field( + target_project: Union[Selector(kind=[ROBOFLOW_PROJECT_KIND]), str] = Field( description="name of Roboflow dataset / project to be used as target for collected data", examples=["my_dataset", "$inputs.target_al_dataset"], ) @@ -166,34 +162,28 @@ class BlockManifest(WorkflowBlockManifest): description="Compression level for images registered", examples=[75], ) - registration_tags: List[ - Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] - ] = Field( + registration_tags: List[Union[Selector(kind=[STRING_KIND]), str]] = Field( default_factory=list, description="Tags to be attached to registered datapoints", examples=[["location-florida", "factory-name", "$inputs.dynamic_tag"]], ) - disable_sink: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( + disable_sink: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=False, description="boolean flag that can be also reference to input - to arbitrarily disable " "data collection for specific request", examples=[True, "$inputs.disable_active_learning"], ) - fire_and_forget: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = ( - Field( - default=True, - description="Boolean flag dictating if sink is supposed to be executed in the background, " - "not waiting on status of registration before end of workflow run. Use `True` if best-effort " - "registration is needed, use `False` while debugging and if error handling is needed", - examples=[True], - ) + fire_and_forget: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Boolean flag dictating if sink is supposed to be executed in the background, " + "not waiting on status of registration before end of workflow run. Use `True` if best-effort " + "registration is needed, use `False` while debugging and if error handling is needed", + examples=[True], ) - labeling_batch_prefix: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = ( - Field( - default="workflows_data_collector", - description="Prefix of the name for labeling batches that will be registered in Roboflow app", - examples=["my_labeling_batch_name"], - ) + labeling_batch_prefix: Union[str, Selector(kind=[STRING_KIND])] = Field( + default="workflows_data_collector", + description="Prefix of the name for labeling batches that will be registered in Roboflow app", + examples=["my_labeling_batch_name"], ) labeling_batches_recreation_frequency: BatchCreationFrequency = Field( default="never", @@ -204,8 +194,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images", "predictions"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -216,7 +206,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RoboflowDatasetUploadBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/sinks/roboflow/dataset_upload/v2.py b/inference/core/workflows/core_steps/sinks/roboflow/dataset_upload/v2.py index efc511200..bbeec7b6a 100644 --- a/inference/core/workflows/core_steps/sinks/roboflow/dataset_upload/v2.py +++ b/inference/core/workflows/core_steps/sinks/roboflow/dataset_upload/v2.py @@ -20,16 +20,14 @@ BOOLEAN_KIND, CLASSIFICATION_PREDICTION_KIND, FLOAT_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, ROBOFLOW_PROJECT_KIND, STRING_KIND, ImageInputField, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -68,10 +66,8 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/roboflow_dataset_upload@v2"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - target_project: Union[ - WorkflowParameterSelector(kind=[ROBOFLOW_PROJECT_KIND]), str - ] = Field( + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + target_project: Union[Selector(kind=[ROBOFLOW_PROJECT_KIND]), str] = Field( description="name of Roboflow dataset / project to be used as target for collected data", examples=["my_dataset", "$inputs.target_al_dataset"], ) @@ -82,7 +78,7 @@ class BlockManifest(WorkflowBlockManifest): json_schema_extra={"hidden": True}, ) predictions: Optional[ - StepOutputSelector( + Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -96,19 +92,15 @@ class BlockManifest(WorkflowBlockManifest): examples=["$steps.object_detection_model.predictions"], json_schema_extra={"always_visible": True}, ) - data_percentage: Union[ - FloatZeroToHundred, WorkflowParameterSelector(kind=[FLOAT_KIND]) - ] = Field( + data_percentage: Union[FloatZeroToHundred, Selector(kind=[FLOAT_KIND])] = Field( default=100, description="Percent of data that will be saved (in range [0.0, 100.0])", examples=[True, False, "$inputs.persist_predictions"], ) - persist_predictions: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = ( - Field( - default=True, - description="Boolean flag to decide if predictions should be registered along with images", - examples=[True, False, "$inputs.persist_predictions"], - ) + persist_predictions: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Boolean flag to decide if predictions should be registered along with images", + examples=[True, False, "$inputs.persist_predictions"], ) minutely_usage_limit: int = Field( default=10, @@ -141,33 +133,27 @@ class BlockManifest(WorkflowBlockManifest): description="Compression level for images registered", examples=[95, 75], ) - registration_tags: List[ - Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] - ] = Field( + registration_tags: List[Union[Selector(kind=[STRING_KIND]), str]] = Field( default_factory=list, description="Tags to be attached to registered datapoints", examples=[["location-florida", "factory-name", "$inputs.dynamic_tag"]], ) - disable_sink: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( + disable_sink: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=False, description="boolean flag that can be also reference to input - to arbitrarily disable " "data collection for specific request", examples=[True, "$inputs.disable_active_learning"], ) - fire_and_forget: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = ( - Field( - default=True, - description="Boolean flag dictating if sink is supposed to be executed in the background, " - "not waiting on status of registration before end of workflow run. Use `True` if best-effort " - "registration is needed, use `False` while debugging and if error handling is needed", - ) + fire_and_forget: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Boolean flag dictating if sink is supposed to be executed in the background, " + "not waiting on status of registration before end of workflow run. Use `True` if best-effort " + "registration is needed, use `False` while debugging and if error handling is needed", ) - labeling_batch_prefix: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = ( - Field( - default="workflows_data_collector", - description="Prefix of the name for labeling batches that will be registered in Roboflow app", - examples=["my_labeling_batch_name"], - ) + labeling_batch_prefix: Union[str, Selector(kind=[STRING_KIND])] = Field( + default="workflows_data_collector", + description="Prefix of the name for labeling batches that will be registered in Roboflow app", + examples=["my_labeling_batch_name"], ) labeling_batches_recreation_frequency: BatchCreationFrequency = Field( default="never", @@ -178,8 +164,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images", "predictions"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -190,7 +176,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RoboflowDatasetUploadBlockV2(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/sinks/webhook/v1.py b/inference/core/workflows/core_steps/sinks/webhook/v1.py index ef3f3a0d9..3652b25fc 100644 --- a/inference/core/workflows/core_steps/sinks/webhook/v1.py +++ b/inference/core/workflows/core_steps/sinks/webhook/v1.py @@ -27,8 +27,7 @@ ROBOFLOW_PROJECT_KIND, STRING_KIND, TOP_CLASS_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -164,7 +163,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/webhook_sink@v1"] - url: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + url: Union[Selector(kind=[STRING_KIND]), str] = Field( description="URL of the resource to make request", ) method: Literal["GET", "POST", "PUT"] = Field( @@ -173,8 +172,8 @@ class BlockManifest(WorkflowBlockManifest): query_parameters: Dict[ str, Union[ - WorkflowParameterSelector(kind=QUERY_PARAMS_KIND), - StepOutputSelector(kind=QUERY_PARAMS_KIND), + Selector(kind=QUERY_PARAMS_KIND), + Selector(kind=QUERY_PARAMS_KIND), str, float, bool, @@ -189,8 +188,8 @@ class BlockManifest(WorkflowBlockManifest): headers: Dict[ str, Union[ - WorkflowParameterSelector(kind=HEADER_KIND), - StepOutputSelector(kind=HEADER_KIND), + Selector(kind=HEADER_KIND), + Selector(kind=HEADER_KIND), str, float, bool, @@ -204,8 +203,8 @@ class BlockManifest(WorkflowBlockManifest): json_payload: Dict[ str, Union[ - WorkflowParameterSelector(), - StepOutputSelector(), + Selector(), + Selector(), str, float, bool, @@ -233,8 +232,8 @@ class BlockManifest(WorkflowBlockManifest): multi_part_encoded_files: Dict[ str, Union[ - WorkflowParameterSelector(), - StepOutputSelector(), + Selector(), + Selector(), str, float, bool, @@ -265,8 +264,8 @@ class BlockManifest(WorkflowBlockManifest): form_data: Dict[ str, Union[ - WorkflowParameterSelector(), - StepOutputSelector(), + Selector(), + Selector(), str, float, bool, @@ -291,35 +290,31 @@ class BlockManifest(WorkflowBlockManifest): ], default_factory=dict, ) - request_timeout: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( + request_timeout: Union[int, Selector(kind=[INTEGER_KIND])] = Field( default=2, description="Number of seconds to wait for remote API response", examples=["$inputs.request_timeout", 10], ) - fire_and_forget: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = ( - Field( - default=True, - description="Boolean flag dictating if sink is supposed to be executed in the background, " - "not waiting on status of registration before end of workflow run. Use `True` if best-effort " - "registration is needed, use `False` while debugging and if error handling is needed", - examples=["$inputs.fire_and_forget", True], - ) + fire_and_forget: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Boolean flag dictating if sink is supposed to be executed in the background, " + "not waiting on status of registration before end of workflow run. Use `True` if best-effort " + "registration is needed, use `False` while debugging and if error handling is needed", + examples=["$inputs.fire_and_forget", True], ) - disable_sink: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( + disable_sink: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( default=False, description="boolean flag that can be also reference to input - to arbitrarily disable " "data collection for specific request", examples=[False, "$inputs.disable_email_notifications"], ) - cooldown_seconds: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = ( - Field( - default=5, - description="Number of seconds to wait until follow-up notification can be sent", - json_schema_extra={ - "always_visible": True, - }, - examples=["$inputs.cooldown_seconds", 10], - ) + cooldown_seconds: Union[int, Selector(kind=[INTEGER_KIND])] = Field( + default=5, + description="Number of seconds to wait until follow-up notification can be sent", + json_schema_extra={ + "always_visible": True, + }, + examples=["$inputs.cooldown_seconds", 10], ) @classmethod @@ -332,7 +327,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class WebhookSinkBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py b/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py index 10ed0031d..755f04d02 100644 --- a/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py +++ b/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py @@ -1,4 +1,3 @@ -from dataclasses import replace from typing import List, Literal, Optional, Type, Union from uuid import uuid4 @@ -6,8 +5,6 @@ from inference.core.workflows.execution_engine.entities.base import ( Batch, - ImageParentMetadata, - OriginCoordinatesSystem, OutputDefinition, WorkflowImageData, ) @@ -15,9 +12,7 @@ IMAGE_KIND, INTEGER_KIND, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -47,31 +42,27 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/absolute_static_crop@v1", "AbsoluteStaticCrop"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - x_center: Union[PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND])] = ( - Field( - description="Center X of static crop (absolute coordinate)", - examples=[40, "$inputs.center_x"], - ) + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + x_center: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + description="Center X of static crop (absolute coordinate)", + examples=[40, "$inputs.center_x"], ) - y_center: Union[PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND])] = ( - Field( - description="Center Y of static crop (absolute coordinate)", - examples=[40, "$inputs.center_y"], - ) + y_center: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + description="Center Y of static crop (absolute coordinate)", + examples=[40, "$inputs.center_y"], ) - width: Union[PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( + width: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( description="Width of static crop (absolute value)", examples=[40, "$inputs.width"], ) - height: Union[PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( + height: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( description="Height of static crop (absolute value)", examples=[40, "$inputs.height"], ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -81,7 +72,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class AbsoluteStaticCropBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/bounding_rect/v1.py b/inference/core/workflows/core_steps/transformations/bounding_rect/v1.py index 49798474a..ad61a3507 100644 --- a/inference/core/workflows/core_steps/transformations/bounding_rect/v1.py +++ b/inference/core/workflows/core_steps/transformations/bounding_rect/v1.py @@ -14,7 +14,7 @@ from inference.core.workflows.execution_engine.entities.base import OutputDefinition from inference.core.workflows.execution_engine.entities.types import ( INSTANCE_SEGMENTATION_PREDICTION_KIND, - StepOutputSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -46,8 +46,8 @@ class BoundingRectManifest(WorkflowBlockManifest): "block_type": "transformation", } ) - type: Literal[f"roboflow_core/bounding_rect@v1"] - predictions: StepOutputSelector( + type: Literal["roboflow_core/bounding_rect@v1"] + predictions: Selector( kind=[ INSTANCE_SEGMENTATION_PREDICTION_KIND, ] @@ -56,10 +56,6 @@ class BoundingRectManifest(WorkflowBlockManifest): examples=["$segmentation.predictions"], ) - @classmethod - def accepts_batch_input(cls) -> bool: - return False - @classmethod def describe_outputs(cls) -> List[OutputDefinition]: return [ @@ -70,7 +66,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" def calculate_minimum_bounding_rectangle( diff --git a/inference/core/workflows/core_steps/transformations/byte_tracker/v1.py b/inference/core/workflows/core_steps/transformations/byte_tracker/v1.py index 3ef4aee75..7ab759eab 100644 --- a/inference/core/workflows/core_steps/transformations/byte_tracker/v1.py +++ b/inference/core/workflows/core_steps/transformations/byte_tracker/v1.py @@ -12,9 +12,8 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputSelector, - WorkflowParameterSelector, - WorkflowVideoMetadataSelector, + VIDEO_METADATA_KIND, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -51,8 +50,8 @@ class ByteTrackerBlockManifest(WorkflowBlockManifest): protected_namespaces=(), ) type: Literal["roboflow_core/byte_tracker@v1"] - metadata: WorkflowVideoMetadataSelector - detections: StepOutputSelector( + metadata: Selector(kind=[VIDEO_METADATA_KIND]) + detections: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -61,28 +60,28 @@ class ByteTrackerBlockManifest(WorkflowBlockManifest): description="Objects to be tracked", examples=["$steps.object_detection_model.predictions"], ) - track_activation_threshold: Union[Optional[float], WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + track_activation_threshold: Union[Optional[float], Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore default=0.25, description="Detection confidence threshold for track activation." " Increasing track_activation_threshold improves accuracy and stability but might miss true detections." " Decreasing it increases completeness but risks introducing noise and instability.", examples=[0.25, "$inputs.confidence"], ) - lost_track_buffer: Union[Optional[int], WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + lost_track_buffer: Union[Optional[int], Selector(kind=[INTEGER_KIND])] = Field( # type: ignore default=30, description="Number of frames to buffer when a track is lost." " Increasing lost_track_buffer enhances occlusion handling, significantly reducing" " the likelihood of track fragmentation or disappearance caused by brief detection gaps.", examples=[30, "$inputs.lost_track_buffer"], ) - minimum_matching_threshold: Union[Optional[float], WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + minimum_matching_threshold: Union[Optional[float], Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore default=0.8, description="Threshold for matching tracks with detections." " Increasing minimum_matching_threshold improves accuracy but risks fragmentation." " Decreasing it improves completeness but risks false positives and drift.", examples=[0.8, "$inputs.min_matching_threshold"], ) - minimum_consecutive_frames: Union[Optional[int], WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + minimum_consecutive_frames: Union[Optional[int], Selector(kind=[INTEGER_KIND])] = Field( # type: ignore default=1, description="Number of consecutive frames that an object must be tracked before it is considered a 'valid' track." " Increasing minimum_consecutive_frames prevents the creation of accidental tracks from false detection" @@ -98,7 +97,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.1.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ByteTrackerBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/byte_tracker/v2.py b/inference/core/workflows/core_steps/transformations/byte_tracker/v2.py index b6ecfab10..0be472e0e 100644 --- a/inference/core/workflows/core_steps/transformations/byte_tracker/v2.py +++ b/inference/core/workflows/core_steps/transformations/byte_tracker/v2.py @@ -13,9 +13,8 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputSelector, + Selector, WorkflowImageSelector, - WorkflowParameterSelector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -58,7 +57,7 @@ class ByteTrackerBlockManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/byte_tracker@v2"] image: WorkflowImageSelector - detections: StepOutputSelector( + detections: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -67,28 +66,28 @@ class ByteTrackerBlockManifest(WorkflowBlockManifest): description="Objects to be tracked", examples=["$steps.object_detection_model.predictions"], ) - track_activation_threshold: Union[Optional[float], WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + track_activation_threshold: Union[Optional[float], Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore default=0.25, description="Detection confidence threshold for track activation." " Increasing track_activation_threshold improves accuracy and stability but might miss true detections." " Decreasing it increases completeness but risks introducing noise and instability.", examples=[0.25, "$inputs.confidence"], ) - lost_track_buffer: Union[Optional[int], WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + lost_track_buffer: Union[Optional[int], Selector(kind=[INTEGER_KIND])] = Field( # type: ignore default=30, description="Number of frames to buffer when a track is lost." " Increasing lost_track_buffer enhances occlusion handling, significantly reducing" " the likelihood of track fragmentation or disappearance caused by brief detection gaps.", examples=[30, "$inputs.lost_track_buffer"], ) - minimum_matching_threshold: Union[Optional[float], WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + minimum_matching_threshold: Union[Optional[float], Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore default=0.8, description="Threshold for matching tracks with detections." " Increasing minimum_matching_threshold improves accuracy but risks fragmentation." " Decreasing it improves completeness but risks false positives and drift.", examples=[0.8, "$inputs.min_matching_threshold"], ) - minimum_consecutive_frames: Union[Optional[int], WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + minimum_consecutive_frames: Union[Optional[int], Selector(kind=[INTEGER_KIND])] = Field( # type: ignore default=1, description="Number of consecutive frames that an object must be tracked before it is considered a 'valid' track." " Increasing minimum_consecutive_frames prevents the creation of accidental tracks from false detection" @@ -104,7 +103,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ByteTrackerBlockV2(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/byte_tracker/v3.py b/inference/core/workflows/core_steps/transformations/byte_tracker/v3.py index 9f0e88ddf..e901ce456 100644 --- a/inference/core/workflows/core_steps/transformations/byte_tracker/v3.py +++ b/inference/core/workflows/core_steps/transformations/byte_tracker/v3.py @@ -11,12 +11,11 @@ ) from inference.core.workflows.execution_engine.entities.types import ( FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -72,8 +71,8 @@ class ByteTrackerBlockManifest(WorkflowBlockManifest): protected_namespaces=(), ) type: Literal["roboflow_core/byte_tracker@v3"] - image: WorkflowImageSelector - detections: StepOutputSelector( + image: Selector(kind=[IMAGE_KIND]) + detections: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -82,28 +81,28 @@ class ByteTrackerBlockManifest(WorkflowBlockManifest): description="Objects to be tracked", examples=["$steps.object_detection_model.predictions"], ) - track_activation_threshold: Union[Optional[float], WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + track_activation_threshold: Union[Optional[float], Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore default=0.25, description="Detection confidence threshold for track activation." " Increasing track_activation_threshold improves accuracy and stability but might miss true detections." " Decreasing it increases completeness but risks introducing noise and instability.", examples=[0.25, "$inputs.confidence"], ) - lost_track_buffer: Union[Optional[int], WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + lost_track_buffer: Union[Optional[int], Selector(kind=[INTEGER_KIND])] = Field( # type: ignore default=30, description="Number of frames to buffer when a track is lost." " Increasing lost_track_buffer enhances occlusion handling, significantly reducing" " the likelihood of track fragmentation or disappearance caused by brief detection gaps.", examples=[30, "$inputs.lost_track_buffer"], ) - minimum_matching_threshold: Union[Optional[float], WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + minimum_matching_threshold: Union[Optional[float], Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore default=0.8, description="Threshold for matching tracks with detections." " Increasing minimum_matching_threshold improves accuracy but risks fragmentation." " Decreasing it improves completeness but risks false positives and drift.", examples=[0.8, "$inputs.min_matching_threshold"], ) - minimum_consecutive_frames: Union[Optional[int], WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + minimum_consecutive_frames: Union[Optional[int], Selector(kind=[INTEGER_KIND])] = Field( # type: ignore default=1, description="Number of consecutive frames that an object must be tracked before it is considered a 'valid' track." " Increasing minimum_consecutive_frames prevents the creation of accidental tracks from false detection" @@ -129,7 +128,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ByteTrackerBlockV3(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/detection_offset/v1.py b/inference/core/workflows/core_steps/transformations/detection_offset/v1.py index 21dbc4f87..9bc42da67 100644 --- a/inference/core/workflows/core_steps/transformations/detection_offset/v1.py +++ b/inference/core/workflows/core_steps/transformations/detection_offset/v1.py @@ -19,8 +19,7 @@ INTEGER_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -51,7 +50,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/detection_offset@v1", "DetectionOffset"] - predictions: StepOutputSelector( + predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -61,24 +60,20 @@ class BlockManifest(WorkflowBlockManifest): description="Reference to detection-like predictions", examples=["$steps.object_detection_model.predictions"], ) - offset_width: Union[PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND])] = ( - Field( - description="Offset for boxes width", - examples=[10, "$inputs.offset_x"], - validation_alias=AliasChoices("offset_width", "offset_x"), - ) + offset_width: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + description="Offset for boxes width", + examples=[10, "$inputs.offset_x"], + validation_alias=AliasChoices("offset_width", "offset_x"), ) - offset_height: Union[ - PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) - ] = Field( + offset_height: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( description="Offset for boxes height", examples=[10, "$inputs.offset_y"], validation_alias=AliasChoices("offset_height", "offset_y"), ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["predictions"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -95,7 +90,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DetectionOffsetBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/detections_filter/v1.py b/inference/core/workflows/core_steps/transformations/detections_filter/v1.py index bae47260b..46a10065c 100644 --- a/inference/core/workflows/core_steps/transformations/detections_filter/v1.py +++ b/inference/core/workflows/core_steps/transformations/detections_filter/v1.py @@ -18,9 +18,7 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -71,7 +69,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/detections_filter@v1", "DetectionsFilter"] - predictions: StepOutputSelector( + predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -86,7 +84,7 @@ class BlockManifest(WorkflowBlockManifest): ) operations_parameters: Dict[ str, - Union[WorkflowImageSelector, WorkflowParameterSelector(), StepOutputSelector()], + Selector(), ] = Field( description="References to additional parameters that may be provided in runtime to parametrise operations", examples=[ @@ -98,8 +96,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["predictions"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -116,7 +114,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DetectionsFilterBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/detections_transformation/v1.py b/inference/core/workflows/core_steps/transformations/detections_transformation/v1.py index 67e6757fc..c947569cd 100644 --- a/inference/core/workflows/core_steps/transformations/detections_transformation/v1.py +++ b/inference/core/workflows/core_steps/transformations/detections_transformation/v1.py @@ -24,9 +24,7 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -85,7 +83,7 @@ class BlockManifest(WorkflowBlockManifest): type: Literal[ "roboflow_core/detections_transformation@v1", "DetectionsTransformation" ] - predictions: StepOutputSelector( + predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -101,7 +99,7 @@ class BlockManifest(WorkflowBlockManifest): ) operations_parameters: Dict[ str, - Union[WorkflowImageSelector, WorkflowParameterSelector(), StepOutputSelector()], + Selector(), ] = Field( description="References to additional parameters that may be provided in runtime to parameterize operations", examples=[ @@ -113,8 +111,12 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["predictions"] + + @classmethod + def get_parameters_accepting_batches_and_scalars(cls) -> List[str]: + return ["operations_parameters"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -131,7 +133,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DetectionsTransformationBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/dynamic_crop/v1.py b/inference/core/workflows/core_steps/transformations/dynamic_crop/v1.py index 60ce2b988..393ed5758 100644 --- a/inference/core/workflows/core_steps/transformations/dynamic_crop/v1.py +++ b/inference/core/workflows/core_steps/transformations/dynamic_crop/v1.py @@ -22,10 +22,7 @@ OBJECT_DETECTION_PREDICTION_KIND, RGB_COLOR_KIND, STRING_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -60,13 +57,13 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/dynamic_crop@v1", "DynamicCrop", "Crop"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + images: Selector(kind=[IMAGE_KIND]) = Field( title="Image to Crop", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], validation_alias=AliasChoices("images", "image"), ) - predictions: StepOutputSelector( + predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -79,7 +76,7 @@ class BlockManifest(WorkflowBlockManifest): validation_alias=AliasChoices("predictions", "detections"), ) mask_opacity: Union[ - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), float, ] = Field( default=0.0, @@ -97,8 +94,8 @@ class BlockManifest(WorkflowBlockManifest): }, ) background_color: Union[ - WorkflowParameterSelector(kind=[STRING_KIND]), - StepOutputSelector(kind=[RGB_COLOR_KIND]), + Selector(kind=[STRING_KIND]), + Selector(kind=[RGB_COLOR_KIND]), str, Tuple[int, int, int], ] = Field( @@ -110,8 +107,8 @@ class BlockManifest(WorkflowBlockManifest): ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images", "predictions"] @classmethod def get_output_dimensionality_offset(cls) -> int: @@ -125,7 +122,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DynamicCropBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/dynamic_zones/v1.py b/inference/core/workflows/core_steps/transformations/dynamic_zones/v1.py index b2dc378ae..f687296a3 100644 --- a/inference/core/workflows/core_steps/transformations/dynamic_zones/v1.py +++ b/inference/core/workflows/core_steps/transformations/dynamic_zones/v1.py @@ -13,8 +13,7 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, LIST_OF_VALUES_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -49,7 +48,7 @@ class DynamicZonesManifest(WorkflowBlockManifest): } ) type: Literal[f"{TYPE}", "DynamicZone"] - predictions: StepOutputSelector( + predictions: Selector( kind=[ INSTANCE_SEGMENTATION_PREDICTION_KIND, ] @@ -57,14 +56,14 @@ class DynamicZonesManifest(WorkflowBlockManifest): description="", examples=["$segmentation.predictions"], ) - required_number_of_vertices: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + required_number_of_vertices: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Keep simplifying polygon until number of vertices matches this number", examples=[4, "$inputs.vertices"], ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["predictions"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -74,7 +73,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" def calculate_simplified_polygon( diff --git a/inference/core/workflows/core_steps/transformations/image_slicer/v1.py b/inference/core/workflows/core_steps/transformations/image_slicer/v1.py index e2ee42fcc..c502212fb 100644 --- a/inference/core/workflows/core_steps/transformations/image_slicer/v1.py +++ b/inference/core/workflows/core_steps/transformations/image_slicer/v1.py @@ -8,8 +8,6 @@ from typing_extensions import Annotated from inference.core.workflows.execution_engine.entities.base import ( - ImageParentMetadata, - OriginCoordinatesSystem, OutputDefinition, WorkflowImageData, ) @@ -17,9 +15,7 @@ FLOAT_ZERO_TO_ONE_KIND, IMAGE_KIND, INTEGER_KIND, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -60,29 +56,25 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/image_slicer@v1"] - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Image to slice", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], validation_alias=AliasChoices("image", "images"), ) - slice_width: Union[PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND])] = ( - Field( - default=640, - description="Width of each slice, in pixels", - examples=[320, "$inputs.slice_width"], - ) + slice_width: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + default=640, + description="Width of each slice, in pixels", + examples=[320, "$inputs.slice_width"], ) - slice_height: Union[PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND])] = ( - Field( - default=640, - description="Height of each slice, in pixels", - examples=[320, "$inputs.slice_height"], - ) + slice_height: Union[PositiveInt, Selector(kind=[INTEGER_KIND])] = Field( + default=640, + description="Height of each slice, in pixels", + examples=[320, "$inputs.slice_height"], ) overlap_ratio_width: Union[ Annotated[float, Field(ge=0.0, lt=1.0)], - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.2, description="Overlap ratio between consecutive slices in the width dimension", @@ -90,7 +82,7 @@ class BlockManifest(WorkflowBlockManifest): ) overlap_ratio_height: Union[ Annotated[float, Field(ge=0.0, lt=1.0)], - WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), ] = Field( default=0.2, description="Overlap ratio between consecutive slices in the height dimension", @@ -109,7 +101,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ImageSlicerBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/perspective_correction/v1.py b/inference/core/workflows/core_steps/transformations/perspective_correction/v1.py index a2fd515dd..c408f0615 100644 --- a/inference/core/workflows/core_steps/transformations/perspective_correction/v1.py +++ b/inference/core/workflows/core_steps/transformations/perspective_correction/v1.py @@ -23,10 +23,7 @@ LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -62,7 +59,7 @@ class PerspectiveCorrectionManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/perspective_correction@v1", "PerspectiveCorrection"] predictions: Optional[ - StepOutputSelector( + Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -73,36 +70,36 @@ class PerspectiveCorrectionManifest(WorkflowBlockManifest): default=None, examples=["$steps.object_detection_model.predictions"], ) - images: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + images: Selector(kind=[IMAGE_KIND]) = Field( title="Image to Crop", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], validation_alias=AliasChoices("images", "image"), ) - perspective_polygons: Union[list, StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore + perspective_polygons: Union[list, Selector(kind=[LIST_OF_VALUES_KIND]), Selector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore description="Perspective polygons (for each batch at least one must be consisting of 4 vertices)", examples=["$steps.perspective_wrap.zones"], ) - transformed_rect_width: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + transformed_rect_width: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Transformed rect width", default=1000, examples=[1000] ) - transformed_rect_height: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + transformed_rect_height: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Transformed rect height", default=1000, examples=[1000] ) - extend_perspective_polygon_by_detections_anchor: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + extend_perspective_polygon_by_detections_anchor: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description=f"If set, perspective polygons will be extended to contain all bounding boxes. Allowed values: {', '.join(sv.Position.list())}", default="", examples=["CENTER"], ) - warp_image: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( # type: ignore + warp_image: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( # type: ignore description=f"If set to True, image will be warped into transformed rect", default=False, examples=[False], ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images", "predictions"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -124,7 +121,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" def pick_largest_perspective_polygons( diff --git a/inference/core/workflows/core_steps/transformations/relative_static_crop/v1.py b/inference/core/workflows/core_steps/transformations/relative_static_crop/v1.py index d0020f927..4a94eaee0 100644 --- a/inference/core/workflows/core_steps/transformations/relative_static_crop/v1.py +++ b/inference/core/workflows/core_steps/transformations/relative_static_crop/v1.py @@ -1,4 +1,3 @@ -from dataclasses import replace from typing import List, Literal, Optional, Type, Union from uuid import uuid4 @@ -6,8 +5,6 @@ from inference.core.workflows.execution_engine.entities.base import ( Batch, - ImageParentMetadata, - OriginCoordinatesSystem, OutputDefinition, WorkflowImageData, ) @@ -16,9 +13,7 @@ IMAGE_KIND, FloatZeroToOne, ImageInputField, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -48,35 +43,27 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/relative_statoic_crop@v1", "RelativeStaticCrop"] - images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField - x_center: Union[ - FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]) - ] = Field( + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + x_center: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( description="Center X of static crop (relative coordinate 0.0-1.0)", examples=[0.3, "$inputs.center_x"], ) - y_center: Union[ - FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]) - ] = Field( + y_center: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( description="Center Y of static crop (relative coordinate 0.0-1.0)", examples=[0.3, "$inputs.center_y"], ) - width: Union[ - FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]) - ] = Field( + width: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( description="Width of static crop (relative value 0.0-1.0)", examples=[0.3, "$inputs.width"], ) - height: Union[ - FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]) - ] = Field( + height: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( description="Height of static crop (relative value 0.0-1.0)", examples=[0.3, "$inputs.height"], ) @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -86,7 +73,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class RelativeStaticCropBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/stabilize_detections/v1.py b/inference/core/workflows/core_steps/transformations/stabilize_detections/v1.py index 5a96b27be..4b75c1bbc 100644 --- a/inference/core/workflows/core_steps/transformations/stabilize_detections/v1.py +++ b/inference/core/workflows/core_steps/transformations/stabilize_detections/v1.py @@ -11,12 +11,11 @@ ) from inference.core.workflows.execution_engine.entities.types import ( FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -46,8 +45,8 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/stabilize_detections@v1"] - image: WorkflowImageSelector - detections: StepOutputSelector( + image: Selector(kind=[IMAGE_KIND]) + detections: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -56,14 +55,14 @@ class BlockManifest(WorkflowBlockManifest): description="Tracked detections", examples=["$steps.object_detection_model.predictions"], ) - smoothing_window_size: Union[Optional[int], WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + smoothing_window_size: Union[Optional[int], Selector(kind=[INTEGER_KIND])] = Field( # type: ignore default=3, description="Predicted movement of detection will be smoothed based on historical measurements of velocity," " this parameter controls number of historical measurements taken under account when calculating smoothed velocity." " Detections will be removed from generating smoothed predictions if they had been missing for longer than this number of frames.", examples=[5, "$inputs.smoothing_window_size"], ) - bbox_smoothing_coefficient: Union[Optional[float], WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + bbox_smoothing_coefficient: Union[Optional[float], Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore default=0.2, description="Bounding box smoothing coefficient applied when given tracker_id is present on current frame." " This parameter must be initialized with value between 0 and 1", @@ -84,7 +83,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class StabilizeTrackedDetectionsBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/stitch_images/v1.py b/inference/core/workflows/core_steps/transformations/stitch_images/v1.py index 63cfa1a2b..d8ff6275b 100644 --- a/inference/core/workflows/core_steps/transformations/stitch_images/v1.py +++ b/inference/core/workflows/core_steps/transformations/stitch_images/v1.py @@ -14,9 +14,7 @@ FLOAT_ZERO_TO_ONE_KIND, IMAGE_KIND, INTEGER_KIND, - StepOutputImageSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -48,26 +46,26 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/stitch_images@v1"] - image1: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image1: Selector(kind=[IMAGE_KIND]) = Field( title="First image to stitch", description="First input image for this step.", examples=["$inputs.image1"], validation_alias=AliasChoices("image1"), ) - image2: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image2: Selector(kind=[IMAGE_KIND]) = Field( title="Second image to stitch", description="Second input image for this step.", examples=["$inputs.image2"], validation_alias=AliasChoices("image2"), ) - max_allowed_reprojection_error: Union[Optional[float], WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + max_allowed_reprojection_error: Union[Optional[float], Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore default=3, description="Advanced parameter overwriting cv.findHomography ransacReprojThreshold parameter." " Maximum allowed reprojection error to treat a point pair as an inlier." " Increasing value of this parameter for low details photo may yield better results.", examples=[3, "$inputs.min_overlap_ratio_w"], ) - count_of_best_matches_per_query_descriptor: Union[Optional[int], WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + count_of_best_matches_per_query_descriptor: Union[Optional[int], Selector(kind=[INTEGER_KIND])] = Field( # type: ignore default=2, description="Advanced parameter overwriting cv.BFMatcher.knnMatch `k` parameter." " Count of best matches found per each query descriptor or less if a query descriptor has less than k possible matches in total.", @@ -82,7 +80,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class StitchImagesBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/transformations/stitch_ocr_detections/v1.py b/inference/core/workflows/core_steps/transformations/stitch_ocr_detections/v1.py index 4141f8de0..6aaed0849 100644 --- a/inference/core/workflows/core_steps/transformations/stitch_ocr_detections/v1.py +++ b/inference/core/workflows/core_steps/transformations/stitch_ocr_detections/v1.py @@ -13,8 +13,7 @@ INTEGER_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -96,7 +95,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/stitch_ocr_detections@v1"] - predictions: StepOutputSelector( + predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, ] @@ -135,7 +134,7 @@ class BlockManifest(WorkflowBlockManifest): } }, ) - tolerance: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( + tolerance: Union[int, Selector(kind=[INTEGER_KIND])] = Field( title="Tolerance", description="The tolerance for grouping detections into the same line of text.", default=10, @@ -154,8 +153,8 @@ def ensure_tolerance_greater_than_zero( return value @classmethod - def accepts_batch_input(cls) -> bool: - return True + def get_parameters_accepting_batches(cls) -> List[str]: + return ["predictions"] @classmethod def describe_outputs(cls) -> List[OutputDefinition]: diff --git a/inference/core/workflows/core_steps/visualizations/background_color/v1.py b/inference/core/workflows/core_steps/visualizations/background_color/v1.py index 3bf9a55d0..d75fdf600 100644 --- a/inference/core/workflows/core_steps/visualizations/background_color/v1.py +++ b/inference/core/workflows/core_steps/visualizations/background_color/v1.py @@ -17,7 +17,7 @@ FLOAT_ZERO_TO_ONE_KIND, STRING_KIND, FloatZeroToOne, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -45,13 +45,13 @@ class BackgroundColorManifest(PredictionsVisualizationManifest): } ) - color: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + color: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description="Color of the background.", default="BLACK", examples=["WHITE", "#FFFFFF", "rgb(255, 255, 255)" "$inputs.background_color"], ) - opacity: Union[FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + opacity: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore description="Transparency of the Mask overlay.", default=0.5, examples=[0.5, "$inputs.opacity"], @@ -59,7 +59,7 @@ class BackgroundColorManifest(PredictionsVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class BackgroundColorVisualizationBlockV1(PredictionsVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/blur/v1.py b/inference/core/workflows/core_steps/visualizations/blur/v1.py index 0f5f3b842..935e0ebe0 100644 --- a/inference/core/workflows/core_steps/visualizations/blur/v1.py +++ b/inference/core/workflows/core_steps/visualizations/blur/v1.py @@ -11,7 +11,7 @@ from inference.core.workflows.execution_engine.entities.base import WorkflowImageData from inference.core.workflows.execution_engine.entities.types import ( INTEGER_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -36,7 +36,7 @@ class BlurManifest(PredictionsVisualizationManifest): } ) - kernel_size: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + kernel_size: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Size of the average pooling kernel used for blurring.", default=15, examples=[15, "$inputs.kernel_size"], @@ -44,7 +44,7 @@ class BlurManifest(PredictionsVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class BlurVisualizationBlockV1(PredictionsVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/bounding_box/v1.py b/inference/core/workflows/core_steps/visualizations/bounding_box/v1.py index 81f892852..11373d882 100644 --- a/inference/core/workflows/core_steps/visualizations/bounding_box/v1.py +++ b/inference/core/workflows/core_steps/visualizations/bounding_box/v1.py @@ -15,7 +15,7 @@ FLOAT_ZERO_TO_ONE_KIND, INTEGER_KIND, FloatZeroToOne, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -40,13 +40,13 @@ class BoundingBoxManifest(ColorableVisualizationManifest): } ) - thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the bounding box in pixels.", default=2, examples=[2, "$inputs.thickness"], ) - roundness: Union[FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + roundness: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore description="Roundness of the corners of the bounding box.", default=0.0, examples=[0.0, "$inputs.roundness"], @@ -54,7 +54,7 @@ class BoundingBoxManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class BoundingBoxVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/circle/v1.py b/inference/core/workflows/core_steps/visualizations/circle/v1.py index c6c1ef067..862d627ae 100644 --- a/inference/core/workflows/core_steps/visualizations/circle/v1.py +++ b/inference/core/workflows/core_steps/visualizations/circle/v1.py @@ -13,7 +13,7 @@ from inference.core.workflows.execution_engine.entities.base import WorkflowImageData from inference.core.workflows.execution_engine.entities.types import ( INTEGER_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -38,7 +38,7 @@ class CircleManifest(ColorableVisualizationManifest): } ) - thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the lines in pixels.", default=2, examples=[2, "$inputs.thickness"], @@ -46,7 +46,7 @@ class CircleManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class CircleVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/color/v1.py b/inference/core/workflows/core_steps/visualizations/color/v1.py index 8b41a8bbc..0dafe27d7 100644 --- a/inference/core/workflows/core_steps/visualizations/color/v1.py +++ b/inference/core/workflows/core_steps/visualizations/color/v1.py @@ -14,7 +14,7 @@ from inference.core.workflows.execution_engine.entities.types import ( FLOAT_ZERO_TO_ONE_KIND, FloatZeroToOne, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -39,7 +39,7 @@ class ColorManifest(ColorableVisualizationManifest): } ) - opacity: Union[FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + opacity: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore description="Transparency of the color overlay.", default=0.5, examples=[0.5, "$inputs.opacity"], @@ -47,7 +47,7 @@ class ColorManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ColorVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/common/base.py b/inference/core/workflows/core_steps/visualizations/common/base.py index bdaa6e0aa..dc6442afa 100644 --- a/inference/core/workflows/core_steps/visualizations/common/base.py +++ b/inference/core/workflows/core_steps/visualizations/common/base.py @@ -14,10 +14,7 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, - StepOutputImageSelector, - StepOutputSelector, - WorkflowImageSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -35,13 +32,13 @@ class VisualizationManifest(WorkflowBlockManifest, ABC): "block_type": "visualization", } ) - image: Union[WorkflowImageSelector, StepOutputImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Input Image", description="The input image for this step.", examples=["$inputs.image", "$steps.cropping.crops"], validation_alias=AliasChoices("image", "images"), ) - copy_image: Union[bool, WorkflowParameterSelector(kind=[BOOLEAN_KIND])] = Field( # type: ignore + copy_image: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( # type: ignore description="Duplicate the image contents (vs overwriting the image in place). Deselect for chained visualizations that should stack on previous ones where the intermediate state is not needed.", default=True, examples=[True, False], @@ -80,7 +77,7 @@ def run( class PredictionsVisualizationManifest(VisualizationManifest, ABC): - predictions: StepOutputSelector( + predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, diff --git a/inference/core/workflows/core_steps/visualizations/common/base_colorable.py b/inference/core/workflows/core_steps/visualizations/common/base_colorable.py index 0d67626d1..bf15aefea 100644 --- a/inference/core/workflows/core_steps/visualizations/common/base_colorable.py +++ b/inference/core/workflows/core_steps/visualizations/common/base_colorable.py @@ -14,7 +14,7 @@ INTEGER_KIND, LIST_OF_VALUES_KIND, STRING_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult @@ -74,7 +74,7 @@ class ColorableVisualizationManifest(PredictionsVisualizationManifest, ABC): # "Matplotlib Oranges_R", # "Matplotlib Reds_R", ], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( # type: ignore default="DEFAULT", description="Color palette to use for annotations.", @@ -83,24 +83,24 @@ class ColorableVisualizationManifest(PredictionsVisualizationManifest, ABC): palette_size: Union[ int, - WorkflowParameterSelector(kind=[INTEGER_KIND]), + Selector(kind=[INTEGER_KIND]), ] = Field( # type: ignore default=10, description="Number of colors in the color palette. Applies when using a matplotlib `color_palette`.", examples=[10, "$inputs.palette_size"], ) - custom_colors: Union[ - List[str], WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]) - ] = Field( # type: ignore - default=[], - description='List of colors to use for annotations when `color_palette` is set to "CUSTOM".', - examples=[["#FF0000", "#00FF00", "#0000FF"], "$inputs.custom_colors"], + custom_colors: Union[List[str], Selector(kind=[LIST_OF_VALUES_KIND])] = ( + Field( # type: ignore + default=[], + description='List of colors to use for annotations when `color_palette` is set to "CUSTOM".', + examples=[["#FF0000", "#00FF00", "#0000FF"], "$inputs.custom_colors"], + ) ) color_axis: Union[ Literal["INDEX", "CLASS", "TRACK"], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( # type: ignore default="CLASS", description="Strategy to use for mapping colors to annotations.", @@ -109,7 +109,7 @@ class ColorableVisualizationManifest(PredictionsVisualizationManifest, ABC): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ColorableVisualizationBlock(PredictionsVisualizationBlock, ABC): diff --git a/inference/core/workflows/core_steps/visualizations/corner/v1.py b/inference/core/workflows/core_steps/visualizations/corner/v1.py index 09cea4966..75ad8e2cb 100644 --- a/inference/core/workflows/core_steps/visualizations/corner/v1.py +++ b/inference/core/workflows/core_steps/visualizations/corner/v1.py @@ -13,7 +13,7 @@ from inference.core.workflows.execution_engine.entities.base import WorkflowImageData from inference.core.workflows.execution_engine.entities.types import ( INTEGER_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -38,13 +38,13 @@ class CornerManifest(ColorableVisualizationManifest): } ) - thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the lines in pixels.", default=4, examples=[4, "$inputs.thickness"], ) - corner_length: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + corner_length: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Length of the corner lines in pixels.", default=15, examples=[15, "$inputs.corner_length"], @@ -52,7 +52,7 @@ class CornerManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class CornerVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/crop/v1.py b/inference/core/workflows/core_steps/visualizations/crop/v1.py index 4390ca4a4..548df08a5 100644 --- a/inference/core/workflows/core_steps/visualizations/crop/v1.py +++ b/inference/core/workflows/core_steps/visualizations/crop/v1.py @@ -15,7 +15,7 @@ FLOAT_KIND, INTEGER_KIND, STRING_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -53,20 +53,20 @@ class CropManifest(ColorableVisualizationManifest): "BOTTOM_RIGHT", "CENTER_OF_MASS", ], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( # type: ignore default="TOP_CENTER", description="The anchor position for placing the crop.", examples=["CENTER", "$inputs.position"], ) - scale_factor: Union[float, WorkflowParameterSelector(kind=[FLOAT_KIND])] = Field( # type: ignore + scale_factor: Union[float, Selector(kind=[FLOAT_KIND])] = Field( # type: ignore description="The factor by which to scale the cropped image part. A factor of 2, for example, would double the size of the cropped area, allowing for a closer view of the detection.", default=2.0, examples=[2.0, "$inputs.scale_factor"], ) - border_thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + border_thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the outline in pixels.", default=2, examples=[2, "$inputs.border_thickness"], @@ -74,7 +74,7 @@ class CropManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class CropVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/dot/v1.py b/inference/core/workflows/core_steps/visualizations/dot/v1.py index c8f76fe99..8504be912 100644 --- a/inference/core/workflows/core_steps/visualizations/dot/v1.py +++ b/inference/core/workflows/core_steps/visualizations/dot/v1.py @@ -14,7 +14,7 @@ from inference.core.workflows.execution_engine.entities.types import ( INTEGER_KIND, STRING_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -54,20 +54,20 @@ class DotManifest(ColorableVisualizationManifest): "BOTTOM_RIGHT", "CENTER_OF_MASS", ], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( # type: ignore default="CENTER", description="The anchor position for placing the dot.", examples=["CENTER", "$inputs.position"], ) - radius: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + radius: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Radius of the dot in pixels.", default=4, examples=[4, "$inputs.radius"], ) - outline_thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + outline_thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the outline of the dot in pixels.", default=0, examples=[2, "$inputs.outline_thickness"], @@ -75,7 +75,7 @@ class DotManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class DotVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/ellipse/v1.py b/inference/core/workflows/core_steps/visualizations/ellipse/v1.py index 3c7624d1d..b9173f36a 100644 --- a/inference/core/workflows/core_steps/visualizations/ellipse/v1.py +++ b/inference/core/workflows/core_steps/visualizations/ellipse/v1.py @@ -13,7 +13,7 @@ from inference.core.workflows.execution_engine.entities.base import WorkflowImageData from inference.core.workflows.execution_engine.entities.types import ( INTEGER_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -38,19 +38,19 @@ class EllipseManifest(ColorableVisualizationManifest): } ) - thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the lines in pixels.", default=2, examples=[2, "$inputs.thickness"], ) - start_angle: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + start_angle: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Starting angle of the ellipse in degrees.", default=-45, examples=[-45, "$inputs.start_angle"], ) - end_angle: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + end_angle: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Ending angle of the ellipse in degrees.", default=235, examples=[235, "$inputs.end_angle"], @@ -58,7 +58,7 @@ class EllipseManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class EllipseVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/halo/v1.py b/inference/core/workflows/core_steps/visualizations/halo/v1.py index 7d74b78f8..1c922380a 100644 --- a/inference/core/workflows/core_steps/visualizations/halo/v1.py +++ b/inference/core/workflows/core_steps/visualizations/halo/v1.py @@ -19,8 +19,7 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, FloatZeroToOne, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -46,7 +45,7 @@ class HaloManifest(ColorableVisualizationManifest): } ) - predictions: StepOutputSelector( + predictions: Selector( kind=[ INSTANCE_SEGMENTATION_PREDICTION_KIND, ] @@ -55,13 +54,13 @@ class HaloManifest(ColorableVisualizationManifest): examples=["$steps.instance_segmentation_model.predictions"], ) - opacity: Union[FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + opacity: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore description="Transparency of the halo overlay.", default=0.8, examples=[0.8, "$inputs.opacity"], ) - kernel_size: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + kernel_size: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Size of the average pooling kernel used for creating the halo.", default=40, examples=[40, "$inputs.kernel_size"], @@ -69,7 +68,7 @@ class HaloManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class HaloVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/keypoint/v1.py b/inference/core/workflows/core_steps/visualizations/keypoint/v1.py index 15a35e179..3bc88b0a2 100644 --- a/inference/core/workflows/core_steps/visualizations/keypoint/v1.py +++ b/inference/core/workflows/core_steps/visualizations/keypoint/v1.py @@ -16,8 +16,7 @@ INTEGER_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, STRING_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -48,7 +47,7 @@ class KeypointManifest(VisualizationManifest): } ) - predictions: StepOutputSelector( + predictions: Selector( kind=[ KEYPOINT_DETECTION_PREDICTION_KIND, ] @@ -63,13 +62,13 @@ class KeypointManifest(VisualizationManifest): json_schema_extra={"always_visible": True}, ) - color: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + color: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description="Color of the keypoint.", default="#A351FB", examples=["#A351FB", "green", "$inputs.color"], ) - text_color: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + text_color: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description="Text color of the keypoint.", default="black", examples=["black", "$inputs.text_color"], @@ -81,7 +80,7 @@ class KeypointManifest(VisualizationManifest): }, }, ) - text_scale: Union[float, WorkflowParameterSelector(kind=[FLOAT_KIND])] = Field( # type: ignore + text_scale: Union[float, Selector(kind=[FLOAT_KIND])] = Field( # type: ignore description="Scale of the text.", default=0.5, examples=[0.5, "$inputs.text_scale"], @@ -94,7 +93,7 @@ class KeypointManifest(VisualizationManifest): }, ) - text_thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + text_thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the text characters.", default=1, examples=[1, "$inputs.text_thickness"], @@ -107,7 +106,7 @@ class KeypointManifest(VisualizationManifest): }, ) - text_padding: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + text_padding: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Padding around the text in pixels.", default=10, examples=[10, "$inputs.text_padding"], @@ -120,7 +119,7 @@ class KeypointManifest(VisualizationManifest): }, ) - thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the outline in pixels.", default=2, examples=[2, "$inputs.thickness"], @@ -133,7 +132,7 @@ class KeypointManifest(VisualizationManifest): }, ) - radius: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + radius: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Radius of the keypoint in pixels.", default=10, examples=[10, "$inputs.radius"], diff --git a/inference/core/workflows/core_steps/visualizations/label/v1.py b/inference/core/workflows/core_steps/visualizations/label/v1.py index b9c46360a..dbff22d2f 100644 --- a/inference/core/workflows/core_steps/visualizations/label/v1.py +++ b/inference/core/workflows/core_steps/visualizations/label/v1.py @@ -16,7 +16,7 @@ FLOAT_KIND, INTEGER_KIND, STRING_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -54,7 +54,7 @@ class LabelManifest(ColorableVisualizationManifest): "Tracker Id", "Time In Zone", ], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( # type: ignore default="Class", description="The type of text to display.", @@ -74,38 +74,38 @@ class LabelManifest(ColorableVisualizationManifest): "BOTTOM_RIGHT", "CENTER_OF_MASS", ], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( # type: ignore default="TOP_LEFT", description="The anchor position for placing the label.", examples=["CENTER", "$inputs.text_position"], ) - text_color: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + text_color: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description="Color of the text.", default="WHITE", examples=["WHITE", "#FFFFFF", "rgb(255, 255, 255)" "$inputs.text_color"], ) - text_scale: Union[float, WorkflowParameterSelector(kind=[FLOAT_KIND])] = Field( # type: ignore + text_scale: Union[float, Selector(kind=[FLOAT_KIND])] = Field( # type: ignore description="Scale of the text.", default=1.0, examples=[1.0, "$inputs.text_scale"], ) - text_thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + text_thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the text characters.", default=1, examples=[1, "$inputs.text_thickness"], ) - text_padding: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + text_padding: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Padding around the text in pixels.", default=10, examples=[10, "$inputs.text_padding"], ) - border_radius: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + border_radius: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Radius of the label in pixels.", default=0, examples=[0, "$inputs.border_radius"], @@ -113,7 +113,7 @@ class LabelManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class LabelVisualizationBlockV1(ColorableVisualizationBlock): @@ -206,7 +206,12 @@ def run( if text == "Class": labels = predictions["class_name"] elif text == "Tracker Id": - labels = [str(t) if t else "" for t in predictions.tracker_id] + if predictions.tracker_id is not None: + labels = [ + str(t) if t else "No Tracker ID" for t in predictions.tracker_id + ] + else: + labels = ["No Tracker ID"] * len(predictions) elif text == "Time In Zone": if "time_in_zone" in predictions.data: labels = [ @@ -241,7 +246,6 @@ def run( labels = [str(d) if d else "" for d in predictions[text]] except Exception: raise ValueError(f"Invalid text type: {text}") - annotated_image = annotator.annotate( scene=image.numpy_image.copy() if copy_image else image.numpy_image, detections=predictions, diff --git a/inference/core/workflows/core_steps/visualizations/line_zone/v1.py b/inference/core/workflows/core_steps/visualizations/line_zone/v1.py index eb3800fcc..77f191f8c 100644 --- a/inference/core/workflows/core_steps/visualizations/line_zone/v1.py +++ b/inference/core/workflows/core_steps/visualizations/line_zone/v1.py @@ -20,8 +20,7 @@ LIST_OF_VALUES_KIND, STRING_KIND, FloatZeroToOne, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -47,43 +46,43 @@ class LineCounterZoneVisualizationManifest(VisualizationManifest): "block_type": "visualization", } ) - zone: Union[list, StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore + zone: Union[list, Selector(kind=[LIST_OF_VALUES_KIND]), Selector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore description="Line in the format [[x1, y1], [x2, y2]] consisting of exactly two points.", examples=[[[0, 50], [500, 50]], "$inputs.zones"], ) - color: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + color: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description="Color of the zone.", default="#5bb573", examples=["WHITE", "#FFFFFF", "rgb(255, 255, 255)" "$inputs.background_color"], ) - thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the lines in pixels.", default=2, examples=[2, "$inputs.thickness"], ) - text_thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + text_thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the text in pixels.", default=1, examples=[1, "$inputs.text_thickness"], ) - text_scale: Union[float, WorkflowParameterSelector(kind=[FLOAT_KIND])] = Field( # type: ignore + text_scale: Union[float, Selector(kind=[FLOAT_KIND])] = Field( # type: ignore description="Scale of the text.", default=1.0, examples=[1.0, "$inputs.text_scale"], ) - count_in: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND]), StepOutputSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + count_in: Union[int, Selector(kind=[INTEGER_KIND]), Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Reference to the number of objects that crossed into the line zone.", default=0, examples=["$steps.line_counter.count_in"], json_schema_extra={"always_visible": True}, ) - count_out: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND]), StepOutputSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + count_out: Union[int, Selector(kind=[INTEGER_KIND]), Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Reference to the number of objects that crossed out of the line zone.", default=0, examples=["$steps.line_counter.count_out"], json_schema_extra={"always_visible": True}, ) - opacity: Union[FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + opacity: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore description="Transparency of the Mask overlay.", default=0.3, examples=[0.3, "$inputs.opacity"], @@ -91,7 +90,7 @@ class LineCounterZoneVisualizationManifest(VisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class LineCounterZoneVisualizationBlockV1(VisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/mask/v1.py b/inference/core/workflows/core_steps/visualizations/mask/v1.py index 975390ea9..8717cb977 100644 --- a/inference/core/workflows/core_steps/visualizations/mask/v1.py +++ b/inference/core/workflows/core_steps/visualizations/mask/v1.py @@ -15,8 +15,7 @@ FLOAT_ZERO_TO_ONE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, FloatZeroToOne, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -42,7 +41,7 @@ class MaskManifest(ColorableVisualizationManifest): } ) - predictions: StepOutputSelector( + predictions: Selector( kind=[ INSTANCE_SEGMENTATION_PREDICTION_KIND, ] @@ -51,7 +50,7 @@ class MaskManifest(ColorableVisualizationManifest): examples=["$steps.instance_segmentation_model.predictions"], ) - opacity: Union[FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + opacity: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore description="Transparency of the Mask overlay.", default=0.5, examples=[0.5, "$inputs.opacity"], @@ -59,7 +58,7 @@ class MaskManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class MaskVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/model_comparison/v1.py b/inference/core/workflows/core_steps/visualizations/model_comparison/v1.py index d3e2484a1..5187a8a34 100644 --- a/inference/core/workflows/core_steps/visualizations/model_comparison/v1.py +++ b/inference/core/workflows/core_steps/visualizations/model_comparison/v1.py @@ -20,8 +20,7 @@ OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, FloatZeroToOne, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -52,7 +51,7 @@ class ModelComparisonManifest(VisualizationManifest): } ) - predictions_a: StepOutputSelector( + predictions_a: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -63,13 +62,13 @@ class ModelComparisonManifest(VisualizationManifest): examples=["$steps.object_detection_model.predictions"], ) - color_a: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + color_a: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description="Color of the areas Model A predicted that Model B did not..", default="GREEN", examples=["GREEN", "#FFFFFF", "rgb(255, 255, 255)" "$inputs.color_a"], ) - predictions_b: StepOutputSelector( + predictions_b: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, @@ -80,19 +79,19 @@ class ModelComparisonManifest(VisualizationManifest): examples=["$steps.object_detection_model.predictions"], ) - color_b: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + color_b: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description="Color of the areas Model B predicted that Model A did not.", default="RED", examples=["RED", "#FFFFFF", "rgb(255, 255, 255)" "$inputs.color_b"], ) - background_color: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + background_color: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description="Color of the areas neither model predicted.", default="BLACK", examples=["BLACK", "#FFFFFF", "rgb(255, 255, 255)" "$inputs.background_color"], ) - opacity: Union[FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + opacity: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore description="Transparency of the overlay.", default=0.7, examples=[0.7, "$inputs.opacity"], @@ -100,7 +99,7 @@ class ModelComparisonManifest(VisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.0.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ModelComparisonVisualizationBlockV1(PredictionsVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/pixelate/v1.py b/inference/core/workflows/core_steps/visualizations/pixelate/v1.py index c00f518d4..40f7e7212 100644 --- a/inference/core/workflows/core_steps/visualizations/pixelate/v1.py +++ b/inference/core/workflows/core_steps/visualizations/pixelate/v1.py @@ -11,7 +11,7 @@ from inference.core.workflows.execution_engine.entities.base import WorkflowImageData from inference.core.workflows.execution_engine.entities.types import ( INTEGER_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -36,7 +36,7 @@ class PixelateManifest(PredictionsVisualizationManifest): } ) - pixel_size: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + pixel_size: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Size of the pixelation.", default=20, examples=[20, "$inputs.pixel_size"], @@ -44,7 +44,7 @@ class PixelateManifest(PredictionsVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class PixelateVisualizationBlockV1(PredictionsVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/polygon/v1.py b/inference/core/workflows/core_steps/visualizations/polygon/v1.py index e93d056cc..b8193c247 100644 --- a/inference/core/workflows/core_steps/visualizations/polygon/v1.py +++ b/inference/core/workflows/core_steps/visualizations/polygon/v1.py @@ -17,8 +17,7 @@ from inference.core.workflows.execution_engine.entities.types import ( INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -44,7 +43,7 @@ class PolygonManifest(ColorableVisualizationManifest): } ) - predictions: StepOutputSelector( + predictions: Selector( kind=[ INSTANCE_SEGMENTATION_PREDICTION_KIND, ] @@ -53,7 +52,7 @@ class PolygonManifest(ColorableVisualizationManifest): examples=["$steps.instance_segmentation_model.predictions"], ) - thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the outline in pixels.", default=2, examples=[2, "$inputs.thickness"], @@ -61,7 +60,7 @@ class PolygonManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class PolygonVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/polygon_zone/v1.py b/inference/core/workflows/core_steps/visualizations/polygon_zone/v1.py index f63cdd1c8..1badaba98 100644 --- a/inference/core/workflows/core_steps/visualizations/polygon_zone/v1.py +++ b/inference/core/workflows/core_steps/visualizations/polygon_zone/v1.py @@ -18,8 +18,7 @@ LIST_OF_VALUES_KIND, STRING_KIND, FloatZeroToOne, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -45,17 +44,17 @@ class PolygonZoneVisualizationManifest(VisualizationManifest): "block_type": "visualization", } ) - zone: Union[list, StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore + zone: Union[list, Selector(kind=[LIST_OF_VALUES_KIND]), Selector(kind=[LIST_OF_VALUES_KIND])] = Field( # type: ignore description="Polygon zones (one for each batch) in a format [[(x1, y1), (x2, y2), (x3, y3), ...], ...];" " each zone must consist of more than 2 points", examples=["$inputs.zones"], ) - color: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + color: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description="Color of the zone.", default="#5bb573", examples=["WHITE", "#FFFFFF", "rgb(255, 255, 255)" "$inputs.background_color"], ) - opacity: Union[FloatZeroToOne, WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + opacity: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore description="Transparency of the Mask overlay.", default=0.3, examples=[0.3, "$inputs.opacity"], @@ -63,7 +62,7 @@ class PolygonZoneVisualizationManifest(VisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class PolygonZoneVisualizationBlockV1(VisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/reference_path/v1.py b/inference/core/workflows/core_steps/visualizations/reference_path/v1.py index 619728b07..6b85d6808 100644 --- a/inference/core/workflows/core_steps/visualizations/reference_path/v1.py +++ b/inference/core/workflows/core_steps/visualizations/reference_path/v1.py @@ -14,8 +14,7 @@ INTEGER_KIND, LIST_OF_VALUES_KIND, STRING_KIND, - StepOutputSelector, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import ( BlockResult, @@ -45,18 +44,18 @@ class ReferencePathVisualizationManifest(VisualizationManifest): ) reference_path: Union[ list, - StepOutputSelector(kind=[LIST_OF_VALUES_KIND]), - WorkflowParameterSelector(kind=[LIST_OF_VALUES_KIND]), + Selector(kind=[LIST_OF_VALUES_KIND]), + Selector(kind=[LIST_OF_VALUES_KIND]), ] = Field( # type: ignore description="Reference path in a format [(x1, y1), (x2, y2), (x3, y3), ...]", examples=["$inputs.expected_path"], ) - color: Union[str, WorkflowParameterSelector(kind=[STRING_KIND])] = Field( # type: ignore + color: Union[str, Selector(kind=[STRING_KIND])] = Field( # type: ignore description="Color of the zone.", default="#5bb573", examples=["WHITE", "#FFFFFF", "rgb(255, 255, 255)" "$inputs.background_color"], ) - thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the lines in pixels.", default=2, examples=[2, "$inputs.thickness"], @@ -73,7 +72,7 @@ def validate_thickness_greater_than_zero( @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class ReferencePathVisualizationBlockV1(WorkflowBlock): diff --git a/inference/core/workflows/core_steps/visualizations/trace/v1.py b/inference/core/workflows/core_steps/visualizations/trace/v1.py index 9d0c7d97f..67332ff4a 100644 --- a/inference/core/workflows/core_steps/visualizations/trace/v1.py +++ b/inference/core/workflows/core_steps/visualizations/trace/v1.py @@ -15,7 +15,7 @@ from inference.core.workflows.execution_engine.entities.types import ( INTEGER_KIND, STRING_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -51,18 +51,18 @@ class TraceManifest(ColorableVisualizationManifest): "BOTTOM_RIGHT", "CENTER_OF_MASS", ], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( # type: ignore default="CENTER", description="The anchor position for placing the label.", examples=["CENTER", "$inputs.text_position"], ) - trace_length: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( + trace_length: Union[int, Selector(kind=[INTEGER_KIND])] = Field( default=30, description="Maximum number of historical tracked objects positions to display.", examples=[30, "$inputs.trace_length"], ) - thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the track visualization line.", default=1, examples=[1, "$inputs.track_thickness"], @@ -77,7 +77,7 @@ def ensure_max_entries_per_file_is_correct(cls, value: Any) -> Any: @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class TraceVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/core_steps/visualizations/triangle/v1.py b/inference/core/workflows/core_steps/visualizations/triangle/v1.py index 1f7230ad0..ce0ecd562 100644 --- a/inference/core/workflows/core_steps/visualizations/triangle/v1.py +++ b/inference/core/workflows/core_steps/visualizations/triangle/v1.py @@ -14,7 +14,7 @@ from inference.core.workflows.execution_engine.entities.types import ( INTEGER_KIND, STRING_KIND, - WorkflowParameterSelector, + Selector, ) from inference.core.workflows.prototypes.block import BlockResult, WorkflowBlockManifest @@ -52,26 +52,26 @@ class TriangleManifest(ColorableVisualizationManifest): "BOTTOM_RIGHT", "CENTER_OF_MASS", ], - WorkflowParameterSelector(kind=[STRING_KIND]), + Selector(kind=[STRING_KIND]), ] = Field( # type: ignore default="TOP_CENTER", description="The anchor position for placing the triangle.", examples=["CENTER", "$inputs.position"], ) - base: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + base: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Base width of the triangle in pixels.", default=10, examples=[10, "$inputs.base"], ) - height: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + height: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Height of the triangle in pixels.", default=10, examples=[10, "$inputs.height"], ) - outline_thickness: Union[int, WorkflowParameterSelector(kind=[INTEGER_KIND])] = Field( # type: ignore + outline_thickness: Union[int, Selector(kind=[INTEGER_KIND])] = Field( # type: ignore description="Thickness of the outline of the triangle in pixels.", default=0, examples=[2, "$inputs.outline_thickness"], @@ -79,7 +79,7 @@ class TriangleManifest(ColorableVisualizationManifest): @classmethod def get_execution_engine_compatibility(cls) -> Optional[str]: - return ">=1.2.0,<2.0.0" + return ">=1.3.0,<2.0.0" class TriangleVisualizationBlockV1(ColorableVisualizationBlock): diff --git a/inference/core/workflows/execution_engine/constants.py b/inference/core/workflows/execution_engine/constants.py index 95c3d619b..055dcc594 100644 --- a/inference/core/workflows/execution_engine/constants.py +++ b/inference/core/workflows/execution_engine/constants.py @@ -1,4 +1,5 @@ NODE_COMPILATION_OUTPUT_PROPERTY = "node_compilation_output" +PARSED_NODE_INPUT_SELECTORS_PROPERTY = "parsed_node_input_selectors" STEP_DEFINITION_PROPERTY = "definition" WORKFLOW_INPUT_BATCH_LINEAGE_ID = "" IMAGE_TYPE_KEY = "type" diff --git a/inference/core/workflows/execution_engine/core.py b/inference/core/workflows/execution_engine/core.py index 955bae3cf..f365af339 100644 --- a/inference/core/workflows/execution_engine/core.py +++ b/inference/core/workflows/execution_engine/core.py @@ -65,11 +65,13 @@ def run( runtime_parameters: Dict[str, Any], fps: float = 0, _is_preview: bool = False, + serialize_results: bool = False, ) -> List[Dict[str, Any]]: return self._engine.run( runtime_parameters=runtime_parameters, fps=fps, _is_preview=_is_preview, + serialize_results=serialize_results, ) diff --git a/inference/core/workflows/execution_engine/entities/base.py b/inference/core/workflows/execution_engine/entities/base.py index d09dccab0..97579f0fe 100644 --- a/inference/core/workflows/execution_engine/entities/base.py +++ b/inference/core/workflows/execution_engine/entities/base.py @@ -55,6 +55,10 @@ def get_type(self) -> str: class WorkflowInput(BaseModel): + type: str + name: str + kind: List[Union[str, Kind]] + dimensionality: int @classmethod def is_batch_oriented(cls) -> bool: @@ -64,7 +68,8 @@ def is_batch_oriented(cls) -> bool: class WorkflowImage(WorkflowInput): type: Literal["WorkflowImage", "InferenceImage"] name: str - kind: List[Kind] = Field(default=[IMAGE_KIND]) + kind: List[Union[str, Kind]] = Field(default=[IMAGE_KIND]) + dimensionality: int = Field(default=1, ge=1, le=1) @classmethod def is_batch_oriented(cls) -> bool: @@ -74,7 +79,19 @@ def is_batch_oriented(cls) -> bool: class WorkflowVideoMetadata(WorkflowInput): type: Literal["WorkflowVideoMetadata"] name: str - kind: List[Kind] = Field(default=[VIDEO_METADATA_KIND]) + kind: List[Union[str, Kind]] = Field(default=[VIDEO_METADATA_KIND]) + dimensionality: int = Field(default=1, ge=1, le=1) + + @classmethod + def is_batch_oriented(cls) -> bool: + return True + + +class WorkflowBatchInput(WorkflowInput): + type: Literal["WorkflowBatchInput"] + name: str + kind: List[Union[str, Kind]] = Field(default_factory=lambda: [WILDCARD_KIND]) + dimensionality: int = Field(default=1) @classmethod def is_batch_oriented(cls) -> bool: @@ -84,14 +101,15 @@ def is_batch_oriented(cls) -> bool: class WorkflowParameter(WorkflowInput): type: Literal["WorkflowParameter", "InferenceParameter"] name: str - kind: List[Kind] = Field(default_factory=lambda: [WILDCARD_KIND]) + kind: List[Union[str, Kind]] = Field(default_factory=lambda: [WILDCARD_KIND]) default_value: Optional[Union[float, int, str, bool, list, set]] = Field( default=None ) + dimensionality: int = Field(default=0, ge=0, le=0) InputType = Annotated[ - Union[WorkflowImage, WorkflowVideoMetadata, WorkflowParameter], + Union[WorkflowImage, WorkflowVideoMetadata, WorkflowParameter, WorkflowBatchInput], Field(discriminator="type"), ] @@ -182,6 +200,10 @@ class VideoMetadata(BaseModel): description="Field represents FPS value (if possible to be retrieved)", default=None, ) + measured_fps: Optional[float] = Field( + description="Field represents measured FPS of live stream", + default=None, + ) comes_from_video_file: Optional[bool] = Field( description="Field is a flag telling if frame comes from video file or stream - " "if not possible to be determined - pass None", @@ -375,7 +397,7 @@ def base64_image(self) -> str: return self._base64_image numpy_image = self.numpy_image self._base64_image = base64.b64encode( - encode_image_to_jpeg_bytes(numpy_image) + encode_image_to_jpeg_bytes(numpy_image, jpeg_quality=95) ).decode("ascii") return self._base64_image diff --git a/inference/core/workflows/execution_engine/entities/engine.py b/inference/core/workflows/execution_engine/entities/engine.py index 1539d0375..e0b7248f4 100644 --- a/inference/core/workflows/execution_engine/entities/engine.py +++ b/inference/core/workflows/execution_engine/entities/engine.py @@ -25,5 +25,6 @@ def run( runtime_parameters: Dict[str, Any], fps: float = 0, _is_preview: bool = False, + serialize_results: bool = False, ) -> List[Dict[str, Any]]: pass diff --git a/inference/core/workflows/execution_engine/entities/types.py b/inference/core/workflows/execution_engine/entities/types.py index 748fe3c52..aac3a76c7 100644 --- a/inference/core/workflows/execution_engine/entities/types.py +++ b/inference/core/workflows/execution_engine/entities/types.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import List, Literal, Optional, Union from pydantic import AliasChoices, BaseModel, Field, StringConstraints from typing_extensions import Annotated @@ -35,6 +35,7 @@ def __hash__(self) -> int: KIND_KEY = "kind" DIMENSIONALITY_OFFSET_KEY = "dimensionality_offset" DIMENSIONALITY_REFERENCE_PROPERTY_KEY = "dimensionality_reference_property" +SELECTOR_POINTS_TO_BATCH_KEY = "selector_points_to_batch" WILDCARD_KIND_DOCS = """ This is a special kind that represents Any value - which is to be used by default if @@ -77,6 +78,7 @@ def __hash__(self) -> int: "video_identifier": "rtsp://some.com/stream1", "comes_from_video_file": False, "fps": 23.99, + "measured_fps": 20.05, "frame_number": 24, "frame_timestamp": "2024-08-21T11:13:44.313999", } @@ -115,6 +117,7 @@ def __hash__(self) -> int: "video_identifier": "rtsp://some.com/stream1", "comes_from_video_file": False, "fps": 23.99, + "measured_fps": 20.05, "frame_number": 24, "frame_timestamp": "2024-08-21T11:13:44.313999", } @@ -1019,9 +1022,30 @@ def __hash__(self) -> int: internal_data_type="str", ) +INFERENCE_ID_KIND_DOCS = """ +This kind represents identifier of inference process, which is usually opaque string used as correlation +identifier for external systems (like Roboflow Model Monitoring). + +Examples: +``` +b1851e3d-a145-4540-a39e-875f21f6cd84 +``` +""" + +INFERENCE_ID_KIND = Kind( + name="inference_id", + description="Inference identifier", + docs=INFERENCE_ID_KIND_DOCS, + serialised_data_type="str", + internal_data_type="str", +) + STEP_AS_SELECTED_ELEMENT = "step" STEP_OUTPUT_AS_SELECTED_ELEMENT = "step_output" +BATCH_AS_SELECTED_ELEMENT = "batch" +SCALAR_AS_SELECTED_ELEMENT = "scalar" +ANY_DATA_AS_SELECTED_ELEMENT = "any_data" StepSelector = Annotated[ str, @@ -1055,6 +1079,7 @@ def StepOutputSelector(kind: Optional[List[Kind]] = None): REFERENCE_KEY: True, SELECTED_ELEMENT_KEY: STEP_OUTPUT_AS_SELECTED_ELEMENT, KIND_KEY: [k.dict() for k in kind], + SELECTOR_POINTS_TO_BATCH_KEY: True, } return Annotated[ str, @@ -1086,6 +1111,7 @@ def WorkflowParameterSelector(kind: Optional[List[Kind]] = None): REFERENCE_KEY: True, SELECTED_ELEMENT_KEY: "workflow_image", KIND_KEY: [IMAGE_KIND.dict()], + SELECTOR_POINTS_TO_BATCH_KEY: True, } ), ] @@ -1098,6 +1124,7 @@ def WorkflowParameterSelector(kind: Optional[List[Kind]] = None): REFERENCE_KEY: True, SELECTED_ELEMENT_KEY: STEP_OUTPUT_AS_SELECTED_ELEMENT, KIND_KEY: [IMAGE_KIND.dict()], + SELECTOR_POINTS_TO_BATCH_KEY: True, } ), ] @@ -1113,6 +1140,27 @@ def WorkflowParameterSelector(kind: Optional[List[Kind]] = None): REFERENCE_KEY: True, SELECTED_ELEMENT_KEY: "workflow_video_metadata", KIND_KEY: [VIDEO_METADATA_KIND.dict()], + SELECTOR_POINTS_TO_BATCH_KEY: True, } ), ] + + +def Selector( + kind: Optional[List[Kind]] = None, +): + if kind is None: + kind = [WILDCARD_KIND] + json_schema_extra = { + REFERENCE_KEY: True, + SELECTED_ELEMENT_KEY: ANY_DATA_AS_SELECTED_ELEMENT, + KIND_KEY: [k.dict() for k in kind], + SELECTOR_POINTS_TO_BATCH_KEY: "dynamic", + } + return Annotated[ + str, + StringConstraints( + pattern=r"(^\$steps\.[A-Za-z_\-0-9]+\.[A-Za-z_*0-9\-]+$)|(^\$inputs.[A-Za-z_0-9\-]+$)" + ), + Field(json_schema_extra=json_schema_extra), + ] diff --git a/inference/core/workflows/execution_engine/introspection/blocks_loader.py b/inference/core/workflows/execution_engine/introspection/blocks_loader.py index dc1bdffe1..7871f40b8 100644 --- a/inference/core/workflows/execution_engine/introspection/blocks_loader.py +++ b/inference/core/workflows/execution_engine/introspection/blocks_loader.py @@ -2,6 +2,7 @@ import logging import os from collections import Counter +from copy import copy from functools import lru_cache from typing import Any, Callable, Dict, List, Optional, Union @@ -9,6 +10,8 @@ from packaging.version import Version from inference.core.workflows.core_steps.loader import ( + KINDS_DESERIALIZERS, + KINDS_SERIALIZERS, REGISTERED_INITIALIZERS, load_blocks, load_kinds, @@ -399,6 +402,90 @@ def _load_plugin_kinds(plugin_name: str) -> List[Kind]: return kinds +@execution_phase( + name="kinds_serializers_loading", + categories=["execution_engine_operation"], +) +def load_kinds_serializers( + profiler: Optional[WorkflowsProfiler] = None, +) -> Dict[str, Callable[[Any], Any]]: + kinds_serializers = copy(KINDS_SERIALIZERS) + plugin_kinds_serializers = load_plugins_serialization_functions( + module_property="KINDS_SERIALIZERS" + ) + kinds_serializers.update(plugin_kinds_serializers) + return kinds_serializers + + +@execution_phase( + name="kinds_deserializers_loading", + categories=["execution_engine_operation"], +) +def load_kinds_deserializers( + profiler: Optional[WorkflowsProfiler] = None, +) -> Dict[str, Callable[[str, Any], Any]]: + kinds_deserializers = copy(KINDS_DESERIALIZERS) + plugin_kinds_deserializers = load_plugins_serialization_functions( + module_property="KINDS_DESERIALIZERS" + ) + kinds_deserializers.update(plugin_kinds_deserializers) + return kinds_deserializers + + +def load_plugins_serialization_functions( + module_property: str, +) -> Dict[str, Callable[[Any], Any]]: + plugins_to_load = get_plugin_modules() + result = {} + for plugin_name in plugins_to_load: + result.update( + load_plugin_serializers( + plugin_name=plugin_name, module_property=module_property + ) + ) + return result + + +def load_plugin_serializers( + plugin_name: str, module_property: str +) -> Dict[str, Callable[[Any], Any]]: + try: + return _load_plugin_serializers( + plugin_name=plugin_name, module_property=module_property + ) + except ImportError as e: + raise PluginLoadingError( + public_message=f"It is not possible to load kinds serializers from workflow plugin `{plugin_name}`. " + f"Make sure the library providing custom step is correctly installed in Python environment.", + context="blocks_loading", + inner_error=e, + ) from e + except AttributeError as e: + raise PluginInterfaceError( + public_message=f"Provided workflow plugin `{plugin_name}` do not implement blocks loading " + f"interface correctly and cannot be loaded.", + context="blocks_loading", + inner_error=e, + ) from e + + +def _load_plugin_serializers( + plugin_name: str, module_property: str +) -> Dict[str, Callable[[Any], Any]]: + module = importlib.import_module(plugin_name) + if not hasattr(module, module_property): + return {} + kinds_serializers = getattr(module, module_property) + if not isinstance(kinds_serializers, dict): + raise PluginInterfaceError( + public_message=f"Provided workflow plugin `{plugin_name}` do not implement blocks loading " + f"interface correctly and cannot be loaded. `{module_property}` is expected to be " + f"dictionary.", + context="blocks_loading", + ) + return kinds_serializers + + def get_plugin_modules() -> List[str]: plugins_to_load = os.environ.get(WORKFLOWS_PLUGINS_ENV) if plugins_to_load is None: diff --git a/inference/core/workflows/execution_engine/introspection/connections_discovery.py b/inference/core/workflows/execution_engine/introspection/connections_discovery.py index 7aec6d53a..a8cd19377 100644 --- a/inference/core/workflows/execution_engine/introspection/connections_discovery.py +++ b/inference/core/workflows/execution_engine/introspection/connections_discovery.py @@ -2,6 +2,8 @@ from typing import Dict, Generator, List, Set, Tuple, Type from inference.core.workflows.execution_engine.entities.types import ( + ANY_DATA_AS_SELECTED_ELEMENT, + BATCH_AS_SELECTED_ELEMENT, STEP_AS_SELECTED_ELEMENT, STEP_OUTPUT_AS_SELECTED_ELEMENT, WILDCARD_KIND, @@ -40,9 +42,14 @@ def discover_blocks_connections( blocks_description=blocks_description, all_schemas=all_schemas, ) + compatible_elements = { + STEP_OUTPUT_AS_SELECTED_ELEMENT, + BATCH_AS_SELECTED_ELEMENT, + ANY_DATA_AS_SELECTED_ELEMENT, + } coarse_input_kind2schemas = convert_kinds_mapping_to_block_wise_format( detailed_input_kind2schemas=detailed_input_kind2schemas, - compatible_elements={STEP_OUTPUT_AS_SELECTED_ELEMENT}, + compatible_elements=compatible_elements, ) input_property_wise_connections = {} output_property_wise_connections = {} @@ -51,6 +58,7 @@ def discover_blocks_connections( starting_block=block_type, all_schemas=all_schemas, output_kind2schemas=output_kind2schemas, + compatible_elements=compatible_elements, ) manifest_type = block_type2manifest_type[block_type] output_property_wise_connections[block_type] = ( @@ -167,12 +175,13 @@ def discover_block_input_connections( starting_block: Type[WorkflowBlock], all_schemas: Dict[Type[WorkflowBlock], BlockManifestMetadata], output_kind2schemas: Dict[str, Set[Type[WorkflowBlock]]], + compatible_elements: Set[str], ) -> Dict[str, Set[Type[WorkflowBlock]]]: result = {} for selector in all_schemas[starting_block].selectors.values(): blocks_matching_property = set() for allowed_reference in selector.allowed_references: - if allowed_reference.selected_element != STEP_OUTPUT_AS_SELECTED_ELEMENT: + if allowed_reference.selected_element not in compatible_elements: continue for single_kind in allowed_reference.kind: blocks_matching_property.update( diff --git a/inference/core/workflows/execution_engine/introspection/entities.py b/inference/core/workflows/execution_engine/introspection/entities.py index 8fba19528..3a8938ad2 100644 --- a/inference/core/workflows/execution_engine/introspection/entities.py +++ b/inference/core/workflows/execution_engine/introspection/entities.py @@ -18,6 +18,7 @@ class ReferenceDefinition: selected_element: str kind: List[Kind] + points_to_batch: Set[bool] @dataclass(frozen=True) diff --git a/inference/core/workflows/execution_engine/introspection/schema_parser.py b/inference/core/workflows/execution_engine/introspection/schema_parser.py index 01976fa56..72d386b36 100644 --- a/inference/core/workflows/execution_engine/introspection/schema_parser.py +++ b/inference/core/workflows/execution_engine/introspection/schema_parser.py @@ -1,12 +1,13 @@ import itertools from collections import OrderedDict, defaultdict from dataclasses import replace -from typing import Dict, Optional, Type +from typing import Dict, Optional, Set, Type from inference.core.workflows.execution_engine.entities.types import ( KIND_KEY, REFERENCE_KEY, SELECTED_ELEMENT_KEY, + SELECTOR_POINTS_TO_BATCH_KEY, Kind, ) from inference.core.workflows.execution_engine.introspection.entities import ( @@ -58,10 +59,16 @@ def parse_block_manifest( dimensionality_reference_property = ( manifest_type.get_dimensionality_reference_property() ) + inputs_accepting_batches = set(manifest_type.get_parameters_accepting_batches()) + inputs_accepting_batches_and_scalars = set( + manifest_type.get_parameters_accepting_batches_and_scalars() + ) return parse_block_manifest_schema( schema=schema, inputs_dimensionality_offsets=inputs_dimensionality_offsets, dimensionality_reference_property=dimensionality_reference_property, + inputs_accepting_batches=inputs_accepting_batches, + inputs_accepting_batches_and_scalars=inputs_accepting_batches_and_scalars, ) @@ -69,6 +76,8 @@ def parse_block_manifest_schema( schema: dict, inputs_dimensionality_offsets: Dict[str, int], dimensionality_reference_property: Optional[str], + inputs_accepting_batches: Set[str], + inputs_accepting_batches_and_scalars: Set[str], ) -> BlockManifestMetadata: primitive_types = retrieve_primitives_from_schema( schema=schema, @@ -77,6 +86,8 @@ def parse_block_manifest_schema( schema=schema, inputs_dimensionality_offsets=inputs_dimensionality_offsets, dimensionality_reference_property=dimensionality_reference_property, + inputs_accepting_batches=inputs_accepting_batches, + inputs_accepting_batches_and_scalars=inputs_accepting_batches_and_scalars, ) return BlockManifestMetadata( primitive_types=primitive_types, @@ -225,6 +236,8 @@ def retrieve_selectors_from_schema( schema: dict, inputs_dimensionality_offsets: Dict[str, int], dimensionality_reference_property: Optional[str], + inputs_accepting_batches: Set[str], + inputs_accepting_batches_and_scalars: Set[str], ) -> Dict[str, SelectorDefinition]: result = [] for property_name, property_definition in schema[PROPERTIES_KEY].items(): @@ -245,6 +258,8 @@ def retrieve_selectors_from_schema( property_dimensionality_offset=property_dimensionality_offset, is_dimensionality_reference_property=is_dimensionality_reference_property, is_list_element=True, + inputs_accepting_batches=inputs_accepting_batches, + inputs_accepting_batches_and_scalars=inputs_accepting_batches_and_scalars, ) elif ( property_definition.get(TYPE_KEY) == OBJECT_TYPE @@ -257,6 +272,8 @@ def retrieve_selectors_from_schema( property_dimensionality_offset=property_dimensionality_offset, is_dimensionality_reference_property=is_dimensionality_reference_property, is_dict_element=True, + inputs_accepting_batches=inputs_accepting_batches, + inputs_accepting_batches_and_scalars=inputs_accepting_batches_and_scalars, ) else: selector = retrieve_selectors_from_simple_property( @@ -265,6 +282,8 @@ def retrieve_selectors_from_schema( property_definition=property_definition, property_dimensionality_offset=property_dimensionality_offset, is_dimensionality_reference_property=is_dimensionality_reference_property, + inputs_accepting_batches=inputs_accepting_batches, + inputs_accepting_batches_and_scalars=inputs_accepting_batches_and_scalars, ) if selector is not None: result.append(selector) @@ -277,10 +296,22 @@ def retrieve_selectors_from_simple_property( property_definition: dict, property_dimensionality_offset: int, is_dimensionality_reference_property: bool, + inputs_accepting_batches: Set[str], + inputs_accepting_batches_and_scalars: Set[str], is_list_element: bool = False, is_dict_element: bool = False, ) -> Optional[SelectorDefinition]: if REFERENCE_KEY in property_definition: + declared_points_to_batch = property_definition.get( + SELECTOR_POINTS_TO_BATCH_KEY, False + ) + if declared_points_to_batch == "dynamic": + if property_name in inputs_accepting_batches_and_scalars: + points_to_batch = {True, False} + else: + points_to_batch = {property_name in inputs_accepting_batches} + else: + points_to_batch = {declared_points_to_batch} allowed_references = [ ReferenceDefinition( selected_element=property_definition[SELECTED_ELEMENT_KEY], @@ -288,6 +319,7 @@ def retrieve_selectors_from_simple_property( Kind.model_validate(k) for k in property_definition.get(KIND_KEY, []) ], + points_to_batch=points_to_batch, ) ] return SelectorDefinition( @@ -309,6 +341,8 @@ def retrieve_selectors_from_simple_property( property_definition=property_definition[ITEMS_KEY], property_dimensionality_offset=property_dimensionality_offset, is_dimensionality_reference_property=is_dimensionality_reference_property, + inputs_accepting_batches=inputs_accepting_batches, + inputs_accepting_batches_and_scalars=inputs_accepting_batches_and_scalars, is_list_element=True, ) if property_defines_union(property_definition=property_definition): @@ -320,6 +354,8 @@ def retrieve_selectors_from_simple_property( is_dict_element=is_dict_element, property_dimensionality_offset=property_dimensionality_offset, is_dimensionality_reference_property=is_dimensionality_reference_property, + inputs_accepting_batches=inputs_accepting_batches, + inputs_accepting_batches_and_scalars=inputs_accepting_batches_and_scalars, ) return None @@ -340,6 +376,8 @@ def retrieve_selectors_from_union_definition( is_dict_element: bool, property_dimensionality_offset: int, is_dimensionality_reference_property: bool, + inputs_accepting_batches: Set[str], + inputs_accepting_batches_and_scalars: Set[str], ) -> Optional[SelectorDefinition]: union_types = ( union_definition.get(ANY_OF_KEY, []) @@ -354,6 +392,8 @@ def retrieve_selectors_from_union_definition( property_definition=type_definition, property_dimensionality_offset=property_dimensionality_offset, is_dimensionality_reference_property=is_dimensionality_reference_property, + inputs_accepting_batches=inputs_accepting_batches, + inputs_accepting_batches_and_scalars=inputs_accepting_batches_and_scalars, is_list_element=is_list_element, ) if result is None: @@ -362,20 +402,27 @@ def retrieve_selectors_from_union_definition( results_references = list( itertools.chain.from_iterable(r.allowed_references for r in results) ) - results_references_by_selected_element = defaultdict(set) + results_references_kind_by_selected_element = defaultdict(set) + results_references_batch_pointing_by_selected_element = defaultdict(set) for reference in results_references: - results_references_by_selected_element[reference.selected_element].update( + results_references_kind_by_selected_element[reference.selected_element].update( reference.kind ) + results_references_batch_pointing_by_selected_element[ + reference.selected_element + ].update(reference.points_to_batch) merged_references = [] for ( reference_selected_element, kind, - ) in results_references_by_selected_element.items(): + ) in results_references_kind_by_selected_element.items(): merged_references.append( ReferenceDefinition( selected_element=reference_selected_element, kind=list(kind), + points_to_batch=results_references_batch_pointing_by_selected_element[ + reference_selected_element + ], ) ) if not merged_references: diff --git a/inference/core/workflows/execution_engine/v1/compiler/core.py b/inference/core/workflows/execution_engine/v1/compiler/core.py index bfbe75441..23d501c4e 100644 --- a/inference/core/workflows/execution_engine/v1/compiler/core.py +++ b/inference/core/workflows/execution_engine/v1/compiler/core.py @@ -9,6 +9,8 @@ from inference.core.workflows.execution_engine.entities.base import WorkflowParameter from inference.core.workflows.execution_engine.introspection.blocks_loader import ( load_initializers, + load_kinds_deserializers, + load_kinds_serializers, load_workflow_blocks, ) from inference.core.workflows.execution_engine.profiling.core import ( @@ -55,6 +57,8 @@ class GraphCompilationResult: parsed_workflow_definition: ParsedWorkflowDefinition available_blocks: List[BlockSpecification] initializers: Dict[str, Union[Any, Callable[[None], Any]]] + kinds_serializers: Dict[str, Callable[[Any], Any]] + kinds_deserializers: Dict[str, Callable[[str, Any], Any]] COMPILATION_CACHE = BasicWorkflowsCache[GraphCompilationResult]( @@ -103,6 +107,8 @@ def compile_workflow( execution_graph=graph_compilation_results.execution_graph, steps=steps_by_name, input_substitutions=input_substitutions, + kinds_serializers=graph_compilation_results.kinds_serializers, + kinds_deserializers=graph_compilation_results.kinds_deserializers, ) @@ -129,6 +135,8 @@ def compile_workflow_graph( profiler=profiler, ) initializers = load_initializers(profiler=profiler) + kinds_serializers = load_kinds_serializers(profiler=profiler) + kinds_deserializers = load_kinds_deserializers(profiler=profiler) dynamic_blocks = compile_dynamic_blocks( dynamic_blocks_definitions=workflow_definition.get( "dynamic_blocks_definitions", [] @@ -154,6 +162,8 @@ def compile_workflow_graph( parsed_workflow_definition=parsed_workflow_definition, available_blocks=available_blocks, initializers=initializers, + kinds_serializers=kinds_serializers, + kinds_deserializers=kinds_deserializers, ) COMPILATION_CACHE.cache(key=key, value=result) return result diff --git a/inference/core/workflows/execution_engine/v1/compiler/entities.py b/inference/core/workflows/execution_engine/v1/compiler/entities.py index 84d0b0606..6c9b945c6 100644 --- a/inference/core/workflows/execution_engine/v1/compiler/entities.py +++ b/inference/core/workflows/execution_engine/v1/compiler/entities.py @@ -1,11 +1,12 @@ from abc import abstractmethod from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, Generator, List, Optional, Set, Type, Union +from typing import Any, Callable, Dict, Generator, List, Optional, Set, Type, Union import networkx as nx from inference.core.workflows.execution_engine.entities.base import InputType, JsonField +from inference.core.workflows.execution_engine.entities.types import WILDCARD_KIND, Kind from inference.core.workflows.execution_engine.introspection.entities import ( ParsedSelector, ) @@ -53,6 +54,12 @@ class CompiledWorkflow: input_substitutions: List[InputSubstitution] workflow_json: Dict[str, Any] init_parameters: Dict[str, Any] + kinds_serializers: Dict[str, Callable[[str, Any], Any]] = field( + default_factory=dict + ) + kinds_deserializers: Dict[str, Callable[[str, Any], Any]] = field( + default_factory=dict + ) class NodeCategory(Enum): @@ -84,6 +91,9 @@ def is_batch_oriented(self) -> bool: @dataclass class OutputNode(ExecutionGraphNode): output_manifest: JsonField + kind: Union[List[Union[Kind, str]], Dict[str, List[Union[Kind, str]]]] = field( + default_factory=lambda: [WILDCARD_KIND] + ) @property def dimensionality(self) -> int: diff --git a/inference/core/workflows/execution_engine/v1/compiler/graph_constructor.py b/inference/core/workflows/execution_engine/v1/compiler/graph_constructor.py index 29d5b5db0..f1a253b41 100644 --- a/inference/core/workflows/execution_engine/v1/compiler/graph_constructor.py +++ b/inference/core/workflows/execution_engine/v1/compiler/graph_constructor.py @@ -1,7 +1,8 @@ import itertools from collections import defaultdict -from copy import copy +from copy import copy, deepcopy from typing import Any, Dict, List, Optional, Set, Tuple, Union +from uuid import uuid4 import networkx as nx from networkx import DiGraph @@ -19,6 +20,7 @@ ) from inference.core.workflows.execution_engine.constants import ( NODE_COMPILATION_OUTPUT_PROPERTY, + PARSED_NODE_INPUT_SELECTORS_PROPERTY, WORKFLOW_INPUT_BATCH_LINEAGE_ID, ) from inference.core.workflows.execution_engine.entities.base import ( @@ -145,11 +147,20 @@ def add_input_nodes_for_graph( ) -> DiGraph: for input_spec in inputs: input_selector = construct_input_selector(input_name=input_spec.name) - data_lineage = ( - [] - if not input_spec.is_batch_oriented() - else [WORKFLOW_INPUT_BATCH_LINEAGE_ID] - ) + if input_spec.is_batch_oriented(): + if input_spec.dimensionality < 1: + raise ExecutionGraphStructureError( + public_message=f"Detected batch oriented input `{input_spec.name}` with " + f"declared dimensionality `{input_spec.dimensionality}` which is below " + f"one (one is minimum dimensionality of the batch). Fix input definition in" + f"your Workflow.", + context="workflow_compilation | execution_graph_construction", + ) + data_lineage = [WORKFLOW_INPUT_BATCH_LINEAGE_ID] + for _ in range(input_spec.dimensionality - 1): + data_lineage.append(f"{uuid4()}") + else: + data_lineage = [] compilation_output = InputNode( node_category=NodeCategory.INPUT_NODE, name=input_spec.name, @@ -215,10 +226,14 @@ def add_steps_edges( execution_graph: DiGraph, ) -> DiGraph: for step in workflow_definition.steps: + source_step_selector = construct_step_selector(step_name=step.name) step_selectors = get_step_selectors(step_manifest=step) + execution_graph.nodes[source_step_selector][ + PARSED_NODE_INPUT_SELECTORS_PROPERTY + ] = step_selectors execution_graph = add_edges_for_step( execution_graph=execution_graph, - step_name=step.name, + source_step_selector=source_step_selector, target_step_parsed_selectors=step_selectors, ) return execution_graph @@ -226,10 +241,9 @@ def add_steps_edges( def add_edges_for_step( execution_graph: DiGraph, - step_name: str, + source_step_selector: str, target_step_parsed_selectors: List[ParsedSelector], ) -> DiGraph: - source_step_selector = construct_step_selector(step_name=step_name) for target_step_parsed_selector in target_step_parsed_selectors: execution_graph = add_edge_for_step( execution_graph=execution_graph, @@ -287,7 +301,8 @@ def add_edge_for_step( f"Failed to validate reference provided for step: {source_step_selector} regarding property: " f"{target_step_parsed_selector.definition.property_name} with value: {target_step_parsed_selector.value}. " f"Allowed kinds of references for this property: {list(set(e.name for e in expected_input_kind))}. " - f"Types of output for referred property: {list(set(a.name for a in actual_input_kind))}" + f"Types of output for referred property: " + f"{list(set(a.name if isinstance(a, Kind) else a for a in actual_input_kind))}" ) validate_reference_kinds( expected=expected_input_kind, @@ -428,22 +443,35 @@ def add_edges_for_outputs( node_selector = get_step_selector_from_its_output( step_output_selector=node_selector ) - output_name = construct_output_selector(name=output.name) + output_selector = construct_output_selector(name=output.name) verify_edge_is_created_between_existing_nodes( execution_graph=execution_graph, start=node_selector, - end=output_name, + end=output_selector, + ) + output_node_manifest = node_as( + execution_graph=execution_graph, + node=output_selector, + expected_type=OutputNode, ) if is_step_output_selector(selector_or_value=output.selector): step_manifest = execution_graph.nodes[node_selector][ NODE_COMPILATION_OUTPUT_PROPERTY ].step_manifest step_outputs = step_manifest.get_actual_outputs() - verify_output_selector_points_to_valid_output( + denote_output_node_kind_based_on_step_outputs( output_selector=output.selector, step_outputs=step_outputs, + output_node_manifest=output_node_manifest, ) - execution_graph.add_edge(node_selector, output_name) + else: + input_manifest = node_as( + execution_graph=execution_graph, + node=node_selector, + expected_type=InputNode, + ).input_manifest + output_node_manifest.kind = copy(input_manifest.kind) + execution_graph.add_edge(node_selector, output_selector) return execution_graph @@ -464,20 +492,24 @@ def verify_edge_is_created_between_existing_nodes( ) -def verify_output_selector_points_to_valid_output( +def denote_output_node_kind_based_on_step_outputs( output_selector: str, step_outputs: List[OutputDefinition], + output_node_manifest: OutputNode, ) -> None: selected_output_name = get_last_chunk_of_selector(selector=output_selector) + kinds_for_outputs = {output.name: output.kind for output in step_outputs} if selected_output_name == "*": + output_node_manifest.kind = deepcopy(kinds_for_outputs) return None - defined_output_names = {output.name for output in step_outputs} - if selected_output_name not in defined_output_names: + if selected_output_name not in kinds_for_outputs: raise InvalidReferenceTargetError( public_message=f"Graph definition contains selector {output_selector} that points to output of step " f"that is not defined in workflow block used to create step.", context="workflow_compilation | execution_graph_construction", ) + output_node_manifest.kind = copy(kinds_for_outputs[selected_output_name]) + return None def denote_data_flow_in_workflow( @@ -642,6 +674,57 @@ def denote_data_flow_for_step( output_dimensionality_offset=output_dimensionality_offset, ) ) + parsed_step_input_selectors: List[ParsedSelector] = execution_graph.nodes[node][ + PARSED_NODE_INPUT_SELECTORS_PROPERTY + ] + input_property2batch_expected = defaultdict(set) + for parsed_selector in parsed_step_input_selectors: + for reference in parsed_selector.definition.allowed_references: + input_property2batch_expected[ + parsed_selector.definition.property_name + ].update(reference.points_to_batch) + for property_name, input_definition in input_data.items(): + if property_name not in input_property2batch_expected: + # only values plugged vi selectors are to be validated + continue + if input_definition.is_compound_input(): + actual_input_is_batch = { + element.is_batch_oriented() + for element in input_definition.iterate_through_definitions() + } + else: + actual_input_is_batch = {input_definition.is_batch_oriented()} + batch_input_expected = input_property2batch_expected[property_name] + step_accepts_batch_input = step_node_data.step_manifest.accepts_batch_input() + if ( + step_accepts_batch_input + and batch_input_expected == {False} + and True in actual_input_is_batch + ): + raise ExecutionGraphStructureError( + public_message=f"Detected invalid reference plugged " + f"into property `{property_name}` of step `{node}` - the step " + f"property do not accept batch-oriented inputs, yet the input selector " + f"holds one - this indicates the problem with " + f"construction of your Workflow - usually the problem occurs when non-batch oriented " + f"step inputs are filled with outputs of batch-oriented steps or batch-oriented inputs.", + context="workflow_compilation | execution_graph_construction", + ) + if ( + step_accepts_batch_input + and batch_input_expected == {True} + and False in actual_input_is_batch + ): + raise ExecutionGraphStructureError( + public_message=f"Detected invalid reference plugged " + f"into property `{property_name}` of step `{node}` - the step " + f"property strictly requires batch-oriented inputs, yet the input selector " + f"holds non-batch oriented input - this indicates the " + f"problem with construction of your Workflow - usually the problem occurs when " + f"non-batch oriented step inputs are filled with outputs of non batch-oriented " + f"steps or non batch-oriented inputs.", + context="workflow_compilation | execution_graph_construction", + ) if not parameters_with_batch_inputs: data_lineage = [] else: diff --git a/inference/core/workflows/execution_engine/v1/compiler/reference_type_checker.py b/inference/core/workflows/execution_engine/v1/compiler/reference_type_checker.py index 845b3aee4..c1733ceb7 100644 --- a/inference/core/workflows/execution_engine/v1/compiler/reference_type_checker.py +++ b/inference/core/workflows/execution_engine/v1/compiler/reference_type_checker.py @@ -1,16 +1,16 @@ -from typing import List +from typing import List, Union from inference.core.workflows.errors import ReferenceTypeError from inference.core.workflows.execution_engine.entities.types import Kind def validate_reference_kinds( - expected: List[Kind], - actual: List[Kind], + expected: List[Union[Kind, str]], + actual: List[Union[Kind, str]], error_message: str, ) -> None: - expected_kind_names = set(e.name for e in expected) - actual_kind_names = set(a.name for a in actual) + expected_kind_names = set(_get_kind_name(kind=e) for e in expected) + actual_kind_names = set(_get_kind_name(kind=a) for a in actual) if "*" in expected_kind_names or "*" in actual_kind_names: return None if len(expected_kind_names.intersection(actual_kind_names)) == 0: @@ -18,3 +18,9 @@ def validate_reference_kinds( public_message=error_message, context="workflow_compilation | execution_graph_construction", ) + + +def _get_kind_name(kind: Union[Kind, str]) -> str: + if isinstance(kind, Kind): + return kind.name + return kind diff --git a/inference/core/workflows/execution_engine/v1/core.py b/inference/core/workflows/execution_engine/v1/core.py index cf683e27c..7135fdb36 100644 --- a/inference/core/workflows/execution_engine/v1/core.py +++ b/inference/core/workflows/execution_engine/v1/core.py @@ -21,7 +21,7 @@ validate_runtime_input, ) -EXECUTION_ENGINE_V1_VERSION = Version("1.2.0") +EXECUTION_ENGINE_V1_VERSION = Version("1.3.0") class ExecutionEngineV1(BaseExecutionEngine): @@ -73,11 +73,13 @@ def run( runtime_parameters: Dict[str, Any], fps: float = 0, _is_preview: bool = False, + serialize_results: bool = False, ) -> List[Dict[str, Any]]: self._profiler.start_workflow_run() runtime_parameters = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=self._compiled_workflow.workflow_definition.inputs, + kinds_deserializers=self._compiled_workflow.kinds_deserializers, prevent_local_images_loading=self._prevent_local_images_loading, profiler=self._profiler, ) @@ -93,6 +95,8 @@ def run( usage_fps=fps, usage_workflow_id=self._workflow_id, usage_workflow_preview=_is_preview, + kinds_serializers=self._compiled_workflow.kinds_serializers, + serialize_results=serialize_results, profiler=self._profiler, ) self._profiler.end_workflow_run() diff --git a/inference/core/workflows/execution_engine/v1/dynamic_blocks/block_assembler.py b/inference/core/workflows/execution_engine/v1/dynamic_blocks/block_assembler.py index 6355ef7c8..46e06046a 100644 --- a/inference/core/workflows/execution_engine/v1/dynamic_blocks/block_assembler.py +++ b/inference/core/workflows/execution_engine/v1/dynamic_blocks/block_assembler.py @@ -13,6 +13,7 @@ from inference.core.workflows.execution_engine.entities.types import ( WILDCARD_KIND, Kind, + Selector, StepOutputImageSelector, StepOutputSelector, WorkflowImageSelector, @@ -249,6 +250,8 @@ def collect_python_types_for_selectors( result.append(WorkflowParameterSelector(kind=selector_kind)) elif selector_type is SelectorType.STEP_OUTPUT: result.append(StepOutputSelector(kind=selector_kind)) + elif selector_type is SelectorType.GENERIC: + result.append(Selector(kind=selector_kind)) else: raise DynamicBlockError( public_message=f"Could not recognise selector type `{selector_type}` declared for input `{input_name}` " @@ -356,8 +359,28 @@ def assembly_manifest_class_methods( describe_outputs = lambda cls: outputs_definitions setattr(manifest_class, "describe_outputs", classmethod(describe_outputs)) setattr(manifest_class, "get_actual_outputs", describe_outputs) - accepts_batch_input = lambda cls: manifest_description.accepts_batch_input + accepts_batch_input = ( + lambda cls: len(manifest_description.batch_oriented_parameters) > 0 + or len(manifest_description.parameters_with_scalars_and_batches) + or manifest_description.accepts_batch_input + ) setattr(manifest_class, "accepts_batch_input", classmethod(accepts_batch_input)) + get_parameters_accepting_batches = ( + lambda cls: manifest_description.batch_oriented_parameters + ) + setattr( + manifest_class, + "get_parameters_accepting_batches", + classmethod(get_parameters_accepting_batches), + ) + get_parameters_accepting_batches_and_scalars = ( + lambda cls: manifest_description.parameters_with_scalars_and_batches + ) + setattr( + manifest_class, + "get_parameters_accepting_batches_and_scalars", + classmethod(get_parameters_accepting_batches_and_scalars), + ) input_dimensionality_offsets = collect_input_dimensionality_offsets( inputs=manifest_description.inputs ) diff --git a/inference/core/workflows/execution_engine/v1/dynamic_blocks/entities.py b/inference/core/workflows/execution_engine/v1/dynamic_blocks/entities.py index 79c52c68a..6e6e6a72f 100644 --- a/inference/core/workflows/execution_engine/v1/dynamic_blocks/entities.py +++ b/inference/core/workflows/execution_engine/v1/dynamic_blocks/entities.py @@ -9,6 +9,7 @@ class SelectorType(Enum): STEP_OUTPUT_IMAGE = "step_output_image" INPUT_PARAMETER = "input_parameter" STEP_OUTPUT = "step_output" + GENERIC = "generic" class ValueType(Enum): @@ -104,6 +105,17 @@ class ManifestDescription(BaseModel): default=False, description="Flag to decide if empty (optional) values will be shipped as run() function parameters", ) + batch_oriented_parameters: List[str] = Field( + default_factory=list, + description="List of batch-oriented parameters. Value will override `accepts_batch_input` if non-empty " + "list is provided, `accepts_batch_input` is kept not to break backward compatibility.", + ) + parameters_with_scalars_and_batches: List[str] = Field( + default_factory=list, + description="List of parameters accepting both batches and scalars at the same time. " + "Value will override `accepts_batch_input` if non-empty " + "list is provided, `accepts_batch_input` is kept not to break backward compatibility.", + ) class PythonCode(BaseModel): diff --git a/inference/core/workflows/execution_engine/v1/executor/core.py b/inference/core/workflows/execution_engine/v1/executor/core.py index f4c86ef86..494c7436c 100644 --- a/inference/core/workflows/execution_engine/v1/executor/core.py +++ b/inference/core/workflows/execution_engine/v1/executor/core.py @@ -1,6 +1,6 @@ from datetime import datetime from functools import partial -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Set from inference.core import logger from inference.core.workflows.errors import ( @@ -44,6 +44,8 @@ def run_workflow( workflow: CompiledWorkflow, runtime_parameters: Dict[str, Any], max_concurrent_steps: int, + kinds_serializers: Optional[Dict[str, Callable[[Any], Any]]], + serialize_results: bool = False, profiler: Optional[WorkflowsProfiler] = None, ) -> List[Dict[str, Any]]: execution_data_manager = ExecutionDataManager.init( @@ -71,6 +73,8 @@ def run_workflow( workflow_outputs=workflow.workflow_definition.outputs, execution_graph=workflow.execution_graph, execution_data_manager=execution_data_manager, + serialize_results=serialize_results, + kinds_serializers=kinds_serializers, ) @@ -194,7 +198,7 @@ def run_simd_step_in_batch_mode( metadata={"step": step_selector}, ): step_input = execution_data_manager.get_simd_step_input( - step_selector=step_selector + step_selector=step_selector, ) with profiler.profile_execution_phase( name="step_code_execution", diff --git a/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/dynamic_batches_manager.py b/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/dynamic_batches_manager.py index cc715cdbd..a35938ea4 100644 --- a/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/dynamic_batches_manager.py +++ b/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/dynamic_batches_manager.py @@ -2,7 +2,7 @@ from networkx import DiGraph -from inference.core.workflows.errors import ExecutionEngineRuntimeError +from inference.core.workflows.errors import AssumptionError, ExecutionEngineRuntimeError from inference.core.workflows.execution_engine.v1.compiler.entities import ( ExecutionGraphNode, InputNode, @@ -83,7 +83,38 @@ def assembly_root_batch_indices( expected_type=InputNode, ) input_parameter_name = input_node_data.input_manifest.name - dimension_value = len(runtime_parameters[input_parameter_name]) - lineage_id = identify_lineage(lineage=node_data.data_lineage) - result[lineage_id] = [(i,) for i in range(dimension_value)] + root_lineage_id = identify_lineage(lineage=node_data.data_lineage[:1]) + result[root_lineage_id] = [ + (i,) for i in range(len(runtime_parameters[input_parameter_name])) + ] + if input_node_data.input_manifest.dimensionality > 1: + lineage_id = identify_lineage(lineage=node_data.data_lineage) + result[lineage_id] = generate_indices_for_input_node( + dimensionality=input_node_data.input_manifest.dimensionality, + dimension_value=runtime_parameters[input_parameter_name], + ) + return result + + +def generate_indices_for_input_node( + dimensionality: int, dimension_value: list, indices_prefix: DynamicBatchIndex = () +) -> List[DynamicBatchIndex]: + if not isinstance(dimension_value, list): + raise AssumptionError( + public_message=f"Could not establish input data batch indices. This is most likely the bug. Contact " + f"Roboflow team through github issues (https://github.com/roboflow/inference/issues) " + f"providing full context of the problem - including workflow definition you use.", + context="workflow_execution | step_input_assembling", + ) + if dimensionality == len(indices_prefix) + 1: + return [indices_prefix + (i,) for i in range(len(dimension_value))] + result = [] + for i, value_element in enumerate(dimension_value): + result.extend( + generate_indices_for_input_node( + dimensionality=dimensionality, + dimension_value=value_element, + indices_prefix=indices_prefix + (i,), + ) + ) return result diff --git a/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/manager.py b/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/manager.py index 349cc1e46..e0c7178f1 100644 --- a/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/manager.py +++ b/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/manager.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Generator, List, Optional, Tuple, Union +from typing import Any, Dict, Generator, List, Optional, Set, Tuple, Union from networkx import DiGraph @@ -149,7 +149,10 @@ def register_non_simd_step_output( outputs=output, ) - def get_simd_step_input(self, step_selector: str) -> BatchModeSIMDStepInput: + def get_simd_step_input( + self, + step_selector: str, + ) -> BatchModeSIMDStepInput: if not self.is_step_simd(step_selector=step_selector): raise ExecutionEngineRuntimeError( public_message=f"Error in execution engine. In context of non-SIMD step: {step_selector} attempts to " diff --git a/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/step_input_assembler.py b/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/step_input_assembler.py index 6bff10c74..cc5b01064 100644 --- a/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/step_input_assembler.py +++ b/inference/core/workflows/execution_engine/v1/executor/execution_data_manager/step_input_assembler.py @@ -326,6 +326,7 @@ def prepare_parameters( result = {} indices_for_parameter = {} guard_of_indices_wrapping = GuardForIndicesWrapping() + compound_inputs = set() for parameter_name, parameter_specs in step_node.input_data.items(): if parameter_specs.is_compound_input(): result[parameter_name], indices_for_parameter[parameter_name] = ( @@ -339,6 +340,7 @@ def prepare_parameters( guard_of_indices_wrapping=guard_of_indices_wrapping, ) ) + compound_inputs.add(parameter_name) else: result[parameter_name], indices_for_parameter[parameter_name] = ( get_non_compound_parameter_value( @@ -438,14 +440,21 @@ def get_non_compound_parameter_value( guard_of_indices_wrapping: GuardForIndicesWrapping, ) -> Union[Any, Optional[List[DynamicBatchIndex]]]: if not parameter.is_batch_oriented(): - input_parameter: DynamicStepInputDefinition = parameter # type: ignore if parameter.points_to_input(): + input_parameter: DynamicStepInputDefinition = parameter # type: ignore parameter_name = get_last_chunk_of_selector( selector=input_parameter.selector ) return runtime_parameters[parameter_name], None - static_input: StaticStepInputDefinition = parameter # type: ignore - return static_input.value, None + elif parameter.points_to_step_output(): + input_parameter: DynamicStepInputDefinition = parameter # type: ignore + value = execution_cache.get_non_batch_output( + selector=input_parameter.selector + ) + return value, None + else: + static_input: StaticStepInputDefinition = parameter # type: ignore + return static_input.value, None dynamic_parameter: DynamicStepInputDefinition = parameter # type: ignore parameter_dimensionality = dynamic_parameter.get_dimensionality() lineage_indices = dynamic_batches_manager.get_indices_for_data_lineage( @@ -454,7 +463,10 @@ def get_non_compound_parameter_value( mask_for_dimension = masks[parameter_dimensionality] if dynamic_parameter.points_to_input(): input_name = get_last_chunk_of_selector(selector=dynamic_parameter.selector) - batch_input = runtime_parameters[input_name] + batch_input = _flatten_batch_oriented_inputs( + runtime_parameters[input_name], + dimensionality=parameter_dimensionality, + ) if mask_for_dimension is not None: if len(lineage_indices) != len(batch_input): raise ExecutionEngineRuntimeError( @@ -516,6 +528,29 @@ def get_non_compound_parameter_value( return result, result.indices +def _flatten_batch_oriented_inputs( + inputs: list, + dimensionality: int, +) -> List[Any]: + if dimensionality == 0 or not isinstance(inputs, list): + raise AssumptionError( + public_message=f"Could not prepare batch-oriented input data. This is most likely the bug. Contact " + f"Roboflow team through github issues (https://github.com/roboflow/inference/issues) " + f"providing full context of the problem - including workflow definition you use.", + context="workflow_execution | step_input_assembling", + ) + if dimensionality == 1: + return inputs + result = [] + for element in inputs: + result.extend( + _flatten_batch_oriented_inputs( + inputs=element, dimensionality=dimensionality - 1 + ) + ) + return result + + def reduce_batch_dimensionality( indices: List[DynamicBatchIndex], upper_level_index: List[DynamicBatchIndex], diff --git a/inference/core/workflows/execution_engine/v1/executor/output_constructor.py b/inference/core/workflows/execution_engine/v1/executor/output_constructor.py index a9b76909f..594bda09d 100644 --- a/inference/core/workflows/execution_engine/v1/executor/output_constructor.py +++ b/inference/core/workflows/execution_engine/v1/executor/output_constructor.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Union import numpy as np import supervision as sv @@ -7,7 +7,7 @@ from inference.core.workflows.core_steps.common.utils import ( sv_detections_to_root_coordinates, ) -from inference.core.workflows.errors import ExecutionEngineRuntimeError +from inference.core.workflows.errors import AssumptionError, ExecutionEngineRuntimeError from inference.core.workflows.execution_engine.constants import ( WORKFLOW_INPUT_BATCH_LINEAGE_ID, ) @@ -15,6 +15,7 @@ CoordinatesSystem, JsonField, ) +from inference.core.workflows.execution_engine.entities.types import WILDCARD_KIND, Kind from inference.core.workflows.execution_engine.v1.compiler.entities import OutputNode from inference.core.workflows.execution_engine.v1.compiler.utils import ( construct_output_selector, @@ -32,6 +33,8 @@ def construct_workflow_output( workflow_outputs: List[JsonField], execution_graph: DiGraph, execution_data_manager: ExecutionDataManager, + serialize_results: bool, + kinds_serializers: Dict[str, Callable[[Any], Any]], ) -> List[Dict[str, Any]]: # Maybe we should make blocks to change coordinates systems: # https://github.com/roboflow/inference/issues/440 @@ -58,6 +61,14 @@ def construct_workflow_output( ).dimensionality for output in workflow_outputs } + kinds_of_output_nodes = { + output.name: node_as( + execution_graph=execution_graph, + node=construct_output_selector(name=output.name), + expected_type=OutputNode, + ).kind + for output in workflow_outputs + } outputs_arrays: Dict[str, Optional[list]] = { name: create_array(indices=np.array(indices)) for name, indices in output_name2indices.items() @@ -87,6 +98,14 @@ def construct_workflow_output( and data_contains_sv_detections(data=data_piece) ): data_piece = convert_sv_detections_coordinates(data=data_piece) + if serialize_results: + output_kind = kinds_of_output_nodes[name] + data_piece = serialize_data_piece( + output_name=name, + data_piece=data_piece, + kind=output_kind, + kinds_serializers=kinds_serializers, + ) try: place_data_in_array( array=array, @@ -152,6 +171,68 @@ def create_empty_index_array(level: int, accumulator: list) -> list: return create_empty_index_array(level - 1, [accumulator]) +def serialize_data_piece( + output_name: str, + data_piece: Any, + kind: Union[List[Union[Kind, str]], Dict[str, List[Union[Kind, str]]]], + kinds_serializers: Dict[str, Callable[[Any], Any]], +) -> Any: + if isinstance(kind, dict): + if not isinstance(data_piece, dict): + raise AssumptionError( + public_message=f"Could not serialize Workflow output `{output_name}` - expected the " + f"output to be dictionary containing all outputs of the step, which is not the case." + f"This is most likely a bug. Contact Roboflow team through github issues " + f"(https://github.com/roboflow/inference/issues) providing full context of" + f"the problem - including workflow definition you use.", + context="workflow_execution | output_construction", + ) + return { + name: serialize_single_workflow_result_field( + output_name=f"{output_name}['{name}']", + value=value, + kind=kind.get(name, [WILDCARD_KIND]), + kinds_serializers=kinds_serializers, + ) + for name, value in data_piece.items() + } + return serialize_single_workflow_result_field( + output_name=output_name, + value=data_piece, + kind=kind, + kinds_serializers=kinds_serializers, + ) + + +def serialize_single_workflow_result_field( + output_name: str, + value: Any, + kind: List[Union[Kind, str]], + kinds_serializers: Dict[str, Callable[[Any], Any]], +) -> Any: + kinds_without_serializer = set() + for single_kind in kind: + kind_name = single_kind.name if isinstance(single_kind, Kind) else kind + serializer = kinds_serializers.get(kind_name) + if serializer is None: + kinds_without_serializer.add(kind_name) + continue + try: + return serializer(value) + except Exception: + # silent exception passing, as it is enough for one serializer to be applied + # for union of kinds + pass + if not kinds_without_serializer: + raise ExecutionEngineRuntimeError( + public_message=f"Requested Workflow output serialization, but for output `{output_name}` which " + f"evaluates into Python type: {type(value)} cannot successfully apply any of " + f"registered serializers.", + context="workflow_execution | output_construction", + ) + return value + + def place_data_in_array(array: list, index: DynamicBatchIndex, data: Any) -> None: if len(index) == 0: raise ExecutionEngineRuntimeError( diff --git a/inference/core/workflows/execution_engine/v1/executor/runtime_input_assembler.py b/inference/core/workflows/execution_engine/v1/executor/runtime_input_assembler.py index f8b36475b..48e8f7e89 100644 --- a/inference/core/workflows/execution_engine/v1/executor/runtime_input_assembler.py +++ b/inference/core/workflows/execution_engine/v1/executor/runtime_input_assembler.py @@ -1,30 +1,13 @@ -import os.path -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple, Union -import cv2 -import numpy as np -from pydantic import ValidationError - -from inference.core.utils.image_utils import ( - attempt_loading_image_from_string, - load_image_from_url, -) -from inference.core.workflows.errors import RuntimeInputError -from inference.core.workflows.execution_engine.entities.base import ( - ImageParentMetadata, - InputType, - VideoMetadata, - WorkflowImage, - WorkflowImageData, - WorkflowVideoMetadata, -) +from inference.core.workflows.errors import AssumptionError, RuntimeInputError +from inference.core.workflows.execution_engine.entities.base import InputType +from inference.core.workflows.execution_engine.entities.types import Kind from inference.core.workflows.execution_engine.profiling.core import ( WorkflowsProfiler, execution_phase, ) -BATCH_ORIENTED_PARAMETER_TYPES = {WorkflowImage, WorkflowVideoMetadata} - @execution_phase( name="workflow_input_assembly", @@ -33,6 +16,7 @@ def assemble_runtime_parameters( runtime_parameters: Dict[str, Any], defined_inputs: List[InputType], + kinds_deserializers: Dict[str, Callable[[str, Any], Any]], prevent_local_images_loading: bool = False, profiler: Optional[WorkflowsProfiler] = None, ) -> Dict[str, Any]: @@ -41,19 +25,14 @@ def assemble_runtime_parameters( defined_inputs=defined_inputs, ) for defined_input in defined_inputs: - if isinstance(defined_input, WorkflowImage): - runtime_parameters[defined_input.name] = assemble_input_image( - parameter=defined_input.name, - image=runtime_parameters.get(defined_input.name), + if defined_input.is_batch_oriented(): + runtime_parameters[defined_input.name] = assemble_batch_oriented_input( + defined_input=defined_input, + value=runtime_parameters.get(defined_input.name), + kinds_deserializers=kinds_deserializers, input_batch_size=input_batch_size, prevent_local_images_loading=prevent_local_images_loading, ) - elif isinstance(defined_input, WorkflowVideoMetadata): - runtime_parameters[defined_input.name] = assemble_video_metadata( - parameter=defined_input.name, - video_metadata=runtime_parameters.get(defined_input.name), - input_batch_size=input_batch_size, - ) else: runtime_parameters[defined_input.name] = assemble_inference_parameter( parameter=defined_input.name, @@ -67,7 +46,7 @@ def determine_input_batch_size( runtime_parameters: Dict[str, Any], defined_inputs: List[InputType] ) -> int: for defined_input in defined_inputs: - if type(defined_input) not in BATCH_ORIENTED_PARAMETER_TYPES: + if not defined_input.is_batch_oriented(): continue parameter_value = runtime_parameters.get(defined_input.name) if isinstance(parameter_value, list) and len(parameter_value) > 1: @@ -75,165 +54,159 @@ def determine_input_batch_size( return 1 -def assemble_input_image( - parameter: str, - image: Any, +def assemble_batch_oriented_input( + defined_input: InputType, + value: Any, + kinds_deserializers: Dict[str, Callable[[str, Any], Any]], input_batch_size: int, - prevent_local_images_loading: bool = False, -) -> List[WorkflowImageData]: - if image is None: + prevent_local_images_loading: bool, +) -> List[Any]: + if value is None: raise RuntimeInputError( - public_message=f"Detected runtime parameter `{parameter}` defined as " - f"`WorkflowImage`, but value is not provided.", + public_message=f"Detected runtime parameter `{defined_input.name}` defined as " + f"`{defined_input.type}` (of kind `{[_get_kind_name(k) for k in defined_input.kind]}`), " + f"but value is not provided.", context="workflow_execution | runtime_input_validation", ) - if not isinstance(image, list): - return [ - _assemble_input_image( - parameter=parameter, - image=image, + if not isinstance(value, list): + result = [ + assemble_single_element_of_batch_oriented_input( + defined_input=defined_input, + value=value, + kinds_deserializers=kinds_deserializers, prevent_local_images_loading=prevent_local_images_loading, ) ] * input_batch_size - result = [ - _assemble_input_image( - parameter=parameter, - image=element, - identifier=idx, - prevent_local_images_loading=prevent_local_images_loading, - ) - for idx, element in enumerate(image) - ] + else: + result = [ + assemble_nested_batch_oriented_input( + current_depth=1, + defined_input=defined_input, + value=element, + kinds_deserializers=kinds_deserializers, + prevent_local_images_loading=prevent_local_images_loading, + identifier=f"{defined_input.name}.[{identifier}]", + ) + for identifier, element in enumerate(value) + ] + if len(result) == 1 and len(result) != input_batch_size: + result = result * input_batch_size if len(result) != input_batch_size: raise RuntimeInputError( public_message="Expected all batch-oriented workflow inputs be the same length, or of length 1 - " - f"but parameter: {parameter} provided with batch size {len(result)}, where expected " + f"but parameter: {defined_input.name} provided with batch size {len(result)}, where expected " f"batch size based on remaining parameters is: {input_batch_size}.", context="workflow_execution | runtime_input_validation", ) return result -def _assemble_input_image( - parameter: str, - image: Any, - identifier: Optional[int] = None, - prevent_local_images_loading: bool = False, -) -> WorkflowImageData: - parent_id = parameter - if identifier is not None: - parent_id = f"{parent_id}.[{identifier}]" - video_metadata = None - if isinstance(image, dict) and "video_metadata" in image: - video_metadata = _assemble_video_metadata( - parameter=parameter, video_metadata=image["video_metadata"] +def assemble_nested_batch_oriented_input( + current_depth: int, + defined_input: InputType, + value: Any, + kinds_deserializers: Dict[str, Callable[[str, Any], Any]], + prevent_local_images_loading: bool, + identifier: Optional[str] = None, +) -> Union[list, Any]: + if current_depth > defined_input.dimensionality: + raise AssumptionError( + public_message=f"While constructing input `{defined_input.name}`, Execution Engine encountered the state " + f"in which it is not possible to construct nested batch-oriented input. " + f"This is most likely the bug. Contact Roboflow team " + f"through github issues (https://github.com/roboflow/inference/issues) providing full " + f"context of the problem - including workflow definition you use.", + context="workflow_execution | step_input_assembling", ) - if isinstance(image, dict) and isinstance(image.get("value"), np.ndarray): - image = image["value"] - if isinstance(image, np.ndarray): - parent_metadata = ImageParentMetadata(parent_id=parent_id) - return WorkflowImageData( - parent_metadata=parent_metadata, - numpy_image=image, - video_metadata=video_metadata, + if current_depth == defined_input.dimensionality: + return assemble_single_element_of_batch_oriented_input( + defined_input=defined_input, + value=value, + kinds_deserializers=kinds_deserializers, + prevent_local_images_loading=prevent_local_images_loading, + identifier=identifier, ) - try: - if isinstance(image, dict): - image = image["value"] - if isinstance(image, str): - base64_image = None - image_reference = None - if image.startswith("http://") or image.startswith("https://"): - image_reference = image - image = load_image_from_url(value=image) - elif not prevent_local_images_loading and os.path.exists(image): - # prevent_local_images_loading is introduced to eliminate - # server vulnerability - namely it prevents local server - # file system from being exploited. - image_reference = image - image = cv2.imread(image) - else: - base64_image = image - image = attempt_loading_image_from_string(image)[0] - parent_metadata = ImageParentMetadata(parent_id=parent_id) - return WorkflowImageData( - parent_metadata=parent_metadata, - numpy_image=image, - base64_image=base64_image, - image_reference=image_reference, - video_metadata=video_metadata, - ) - except Exception as error: + if not isinstance(value, list): raise RuntimeInputError( - public_message=f"Detected runtime parameter `{parameter}` defined as `WorkflowImage` " - f"that is invalid. Failed on input validation. Details: {error}", + public_message=f"Workflow input `{defined_input.name}` is declared to be nested batch with dimensionality " + f"`{defined_input.dimensionality}`. Input data does not define batch at the {current_depth} " + f"dimensionality level.", context="workflow_execution | runtime_input_validation", - ) from error + ) + return [ + assemble_nested_batch_oriented_input( + current_depth=current_depth + 1, + defined_input=defined_input, + value=element, + kinds_deserializers=kinds_deserializers, + prevent_local_images_loading=prevent_local_images_loading, + identifier=f"{identifier}.[{idx}]", + ) + for idx, element in enumerate(value) + ] + + +def assemble_single_element_of_batch_oriented_input( + defined_input: InputType, + value: Any, + kinds_deserializers: Dict[str, Callable[[str, Any], Any]], + prevent_local_images_loading: bool, + identifier: Optional[str] = None, +) -> Any: + if value is None: + return None + matching_deserializers = _get_matching_deserializers( + defined_input=defined_input, + kinds_deserializers=kinds_deserializers, + ) + if not matching_deserializers: + return value + parameter_identifier = defined_input.name + if identifier is not None: + parameter_identifier = identifier + errors = [] + for kind, deserializer in matching_deserializers: + try: + if kind == "image": + # this is left-over of bad design decision with adding `prevent_local_images_loading` + # flag at the level of execution engine. To avoid BC we need to + # be aware of special treatment for image kind. + # TODO: deprecate in v2 of Execution Engine + return deserializer( + parameter_identifier, value, prevent_local_images_loading + ) + return deserializer(parameter_identifier, value) + except Exception as error: + errors.append((kind, error)) + error_message = ( + f"Failed to assemble `{parameter_identifier}`. " + f"Could not successfully use any deserializer for declared kinds. Details: " + ) + for kind, error in errors: + error_message = f"{error_message}\nKind: `{kind}` - Error: {error}" raise RuntimeInputError( - public_message=f"Detected runtime parameter `{parameter}` defined as `WorkflowImage` " - f"with type {type(image)} that is invalid. Workflows accept only np.arrays " - f"and dicts with keys `type` and `value` compatible with `inference` (or list of them).", + public_message=error_message, context="workflow_execution | runtime_input_validation", ) -def assemble_video_metadata( - parameter: str, - video_metadata: Any, - input_batch_size: int, -) -> List[VideoMetadata]: - if video_metadata is None: - raise RuntimeInputError( - public_message=f"Detected runtime parameter `{parameter}` defined as " - f"`WorkflowVideoMetadata`, but value is not provided.", - context="workflow_execution | runtime_input_validation", - ) - if not isinstance(video_metadata, list): - return [ - _assemble_video_metadata( - parameter=parameter, - video_metadata=video_metadata, - ) - ] * input_batch_size - result = [ - _assemble_video_metadata( - parameter=parameter, - video_metadata=element, - ) - for element in video_metadata - ] - if len(result) != input_batch_size: - raise RuntimeInputError( - public_message="Expected all batch-oriented workflow inputs be the same length, or of length 1 - " - f"but parameter: {parameter} provided with batch size {len(result)}, where expected " - f"batch size based on remaining parameters is: {input_batch_size}.", - context="workflow_execution | runtime_input_validation", - ) - return result +def _get_matching_deserializers( + defined_input: InputType, + kinds_deserializers: Dict[str, Callable[[str, Any], Any]], +) -> List[Tuple[str, Callable[[str, Any], Any]]]: + matching_deserializers = [] + for kind in defined_input.kind: + kind_name = _get_kind_name(kind=kind) + if kind_name not in kinds_deserializers: + continue + matching_deserializers.append((kind_name, kinds_deserializers[kind_name])) + return matching_deserializers -def _assemble_video_metadata( - parameter: str, - video_metadata: Any, -) -> VideoMetadata: - if isinstance(video_metadata, VideoMetadata): - return video_metadata - if not isinstance(video_metadata, dict): - raise RuntimeInputError( - public_message=f"Detected runtime parameter `{parameter}` holding " - f"`WorkflowVideoMetadata`, but provided value is not a dict.", - context="workflow_execution | runtime_input_validation", - ) - try: - return VideoMetadata.model_validate(video_metadata) - except ValidationError as error: - raise RuntimeInputError( - public_message=f"Detected runtime parameter `{parameter}` holding " - f"`WorkflowVideoMetadata`, but provided value is malformed. " - f"See details in inner error.", - context="workflow_execution | runtime_input_validation", - inner_error=error, - ) +def _get_kind_name(kind: Union[Kind, str]) -> str: + if isinstance(kind, Kind): + return kind.name + return kind def assemble_inference_parameter( diff --git a/inference/core/workflows/execution_engine/v1/introspection/inputs_discovery.py b/inference/core/workflows/execution_engine/v1/introspection/inputs_discovery.py index 1037c7f18..b660870b4 100644 --- a/inference/core/workflows/execution_engine/v1/introspection/inputs_discovery.py +++ b/inference/core/workflows/execution_engine/v1/introspection/inputs_discovery.py @@ -36,11 +36,26 @@ "workflow_video_metadata": {"WorkflowVideoMetadata"}, "workflow_image": {"WorkflowImage", "InferenceImage"}, "workflow_parameter": {"WorkflowParameter", "InferenceParameter"}, + "any_data": { + "WorkflowVideoMetadata", + "WorkflowImage", + "InferenceImage", + "WorkflowBatchInput", + "WorkflowParameter", + "InferenceParameter", + }, } INPUT_TYPE_TO_SELECTED_ELEMENT = { - input_type: selected_element - for selected_element, input_types in SELECTED_ELEMENT_TO_INPUT_TYPE.items() - for input_type in input_types + "WorkflowVideoMetadata": {"workflow_video_metadata", "any_data"}, + "WorkflowImage": {"workflow_image", "any_data"}, + "InferenceImage": {"workflow_image", "any_data"}, + "WorkflowParameter": {"workflow_parameter", "any_data"}, + "InferenceParameter": {"workflow_parameter", "any_data"}, + "WorkflowBatchInput": { + "workflow_image", + "workflow_video_metadata", + "any_data", + }, } @@ -222,8 +237,6 @@ def grab_input_compatible_references_kinds( ) -> Dict[str, Set[str]]: matching_references = defaultdict(set) for reference in selector_definition.allowed_references: - if reference.selected_element not in SELECTED_ELEMENT_TO_INPUT_TYPE: - continue matching_references[reference.selected_element].update( k.name for k in reference.kind ) @@ -275,8 +288,11 @@ def prepare_search_results_for_detected_selectors( f"which is not supported in this installation of Workflow Execution Engine.", context="describing_workflow_inputs", ) - selected_element = INPUT_TYPE_TO_SELECTED_ELEMENT[selector_details.type] - kinds_for_element = matching_references_kinds[selected_element] + + selected_elements = INPUT_TYPE_TO_SELECTED_ELEMENT[selector_details.type] + kinds_for_element = set() + for selected_element in selected_elements: + kinds_for_element.update(matching_references_kinds[selected_element]) if not kinds_for_element: raise WorkflowDefinitionError( public_message=f"Workflow definition invalid - selector `{detected_input_selector}` declared for " diff --git a/inference/core/workflows/prototypes/block.py b/inference/core/workflows/prototypes/block.py index bdc9f644f..6ad5db2dc 100644 --- a/inference/core/workflows/prototypes/block.py +++ b/inference/core/workflows/prototypes/block.py @@ -55,7 +55,17 @@ def get_output_dimensionality_offset( @classmethod def accepts_batch_input(cls) -> bool: - return False + return len(cls.get_parameters_accepting_batches()) > 0 or len( + cls.get_parameters_accepting_batches_and_scalars() + ) + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return [] + + @classmethod + def get_parameters_accepting_batches_and_scalars(cls) -> List[str]: + return [] @classmethod def accepts_empty_values(cls) -> bool: diff --git a/inference/models/transformers/transformers.py b/inference/models/transformers/transformers.py index 8a1ed382b..15043da48 100644 --- a/inference/models/transformers/transformers.py +++ b/inference/models/transformers/transformers.py @@ -126,10 +126,12 @@ def predict(self, image_in: Image.Image, prompt="", history=None, **kwargs): max_new_tokens=1000, do_sample=False, early_stopping=False, + no_repeat_ngram_size=0, ) generation = generation[0] if self.generation_includes_input: generation = generation[input_len:] + decoded = self.processor.decode( generation, skip_special_tokens=self.skip_special_tokens ) @@ -151,9 +153,8 @@ def get_infer_bucket_file_list(self) -> list: "config.json", "special_tokens_map.json", "generation_config.json", - "model.safetensors.index.json", "tokenizer.json", - re.compile(r"model-\d{5}-of-\d{5}\.safetensors"), + re.compile(r"model.*\.safetensors"), "preprocessor_config.json", "tokenizer_config.json", ] @@ -181,16 +182,21 @@ def download_model_artifacts_from_roboflow_api(self) -> None: model_id=self.endpoint, ) if filename.endswith("tar.gz"): - subprocess.run( - [ - "tar", - "-xzf", - os.path.join(self.cache_dir, filename), - "-C", - self.cache_dir, - ], - check=True, - ) + try: + subprocess.run( + [ + "tar", + "-xzf", + os.path.join(self.cache_dir, filename), + "-C", + self.cache_dir, + ], + check=True, + ) + except subprocess.CalledProcessError as e: + raise ModelArtefactError( + f"Failed to extract model archive {filename}. Error: {str(e)}" + ) from e if perf_counter() - t1 > 120: logger.debug( @@ -286,7 +292,6 @@ def get_infer_bucket_file_list(self) -> list: "adapter_config.json", "special_tokens_map.json", "tokenizer.json", - "tokenizer.model", "adapter_model.safetensors", "preprocessor_config.json", "tokenizer_config.json", diff --git a/inference/models/yolo_world/yolo_world.py b/inference/models/yolo_world/yolo_world.py index a5d790440..3209b41e8 100644 --- a/inference/models/yolo_world/yolo_world.py +++ b/inference/models/yolo_world/yolo_world.py @@ -5,7 +5,7 @@ import clip import numpy as np import torch -from ultralytics import YOLO +from ultralytics import YOLO, settings from inference.core import logger from inference.core.cache import cache @@ -30,6 +30,9 @@ EMBEDDINGS_EXPIRE_TIMEOUT = 1800 # 30 min +settings.update({"sync": False}) + + class YOLOWorld(RoboflowCoreModel): """YOLO-World class for zero-shot object detection. diff --git a/inference/usage_tracking/collector.py b/inference/usage_tracking/collector.py index 5f5ff3e3e..2c587c157 100644 --- a/inference/usage_tracking/collector.py +++ b/inference/usage_tracking/collector.py @@ -2,6 +2,7 @@ import atexit import json import mimetypes +import numbers import socket import sys import time @@ -315,6 +316,7 @@ def _update_usage_payload( fps: float = 0, ): source = str(source) if source else "" + frames = frames if isinstance(frames, numbers.Number) else 0 api_key_hash = self._calculate_api_key_hash(api_key=api_key) if not resource_id and resource_details: resource_id = UsageCollector._calculate_resource_hash(resource_details) @@ -332,7 +334,9 @@ def _update_usage_payload( source_usage["timestamp_start"] = time.time_ns() source_usage["timestamp_stop"] = time.time_ns() source_usage["processed_frames"] += frames if not inference_test_run else 0 - source_usage["fps"] = round(fps, 2) + source_usage["fps"] = ( + round(fps, 2) if isinstance(fps, numbers.Number) else 0 + ) source_usage["source_duration"] += ( frames / fps if fps and not inference_test_run else 0 ) @@ -355,7 +359,7 @@ def record_usage( resource_id: str = "", inference_test_run: bool = False, fps: float = 0, - ) -> DefaultDict[str, Any]: + ): if not api_key: return if self._settings.opt_out and not api_key: @@ -388,7 +392,7 @@ async def async_record_usage( resource_id: str = "", inference_test_run: bool = False, fps: float = 0, - ) -> DefaultDict[str, Any]: + ): if self._async_lock: async with self._async_lock: self.record_usage( diff --git a/inference_cli/configs/bounding_boxes_tracing.yml b/inference_cli/configs/bounding_boxes_tracing.yml index 019db48ef..3b8671e2b 100644 --- a/inference_cli/configs/bounding_boxes_tracing.yml +++ b/inference_cli/configs/bounding_boxes_tracing.yml @@ -12,7 +12,7 @@ annotators: trace_length: 60 thickness: 2 tracking: - track_thresh: 0.25 - track_buffer: 30 - match_thresh: 0.8 + track_activation_threshold: 0.25 + lost_track_buffer: 30 + minimum_matching_threshold: 0.8 frame_rate: 30 diff --git a/inference_cli/lib/cloud_adapter.py b/inference_cli/lib/cloud_adapter.py index 82f47e034..8f6e7a607 100644 --- a/inference_cli/lib/cloud_adapter.py +++ b/inference_cli/lib/cloud_adapter.py @@ -83,14 +83,18 @@ """, } + def check_sky_installed(): try: global sky import sky except ImportError as e: - print("Please install cloud deploy dependencies with 'pip install inference[cloud-deploy]'") + print( + "Please install cloud deploy dependencies with 'pip install inference[cloud-deploy]'" + ) raise e + def _random_char(y): return "".join(random.choice(string.ascii_lowercase) for x in range(y)) diff --git a/inference_cli/lib/infer_adapter.py b/inference_cli/lib/infer_adapter.py index 15f1940d6..1fa5af632 100644 --- a/inference_cli/lib/infer_adapter.py +++ b/inference_cli/lib/infer_adapter.py @@ -8,7 +8,7 @@ import numpy as np from supervision import ( BlurAnnotator, - BoundingBoxAnnotator, + BoxAnnotator, BoxCornerAnnotator, ByteTrack, CircleAnnotator, @@ -46,7 +46,8 @@ ) ANNOTATOR_TYPE2CLASS = { - "bounding_box": BoundingBoxAnnotator, + "bounding_box": BoxAnnotator, + "box": BoxAnnotator, "mask": MaskAnnotator, "polygon": PolygonAnnotator, "color": ColorAnnotator, @@ -313,7 +314,7 @@ def is_something_to_do( def build_visualisation_callback( visualisation_config: Optional[str], ) -> Callable[[np.ndarray, dict], Optional[np.ndarray]]: - annotators = [BoundingBoxAnnotator()] + annotators = [BoxAnnotator()] byte_tracker = None if visualisation_config is not None: raw_configuration = retrieve_visualisation_config( diff --git a/inference_sdk/http/client.py b/inference_sdk/http/client.py index c87af9e59..085ce24d8 100644 --- a/inference_sdk/http/client.py +++ b/inference_sdk/http/client.py @@ -42,6 +42,7 @@ ) from inference_sdk.http.utils.iterables import unwrap_single_element_list from inference_sdk.http.utils.loaders import ( + load_nested_batches_of_inference_input, load_static_inference_input, load_static_inference_input_async, load_stream_inference_input, @@ -65,6 +66,7 @@ api_key_safe_raise_for_status, deduct_api_key_from_string, inject_images_into_payload, + inject_nested_batches_of_images_into_payload, ) from inference_sdk.utils.decorators import deprecated, experimental @@ -1156,10 +1158,10 @@ def _run_workflow( } inputs = {} for image_name, image in images.items(): - loaded_image = load_static_inference_input( + loaded_image = load_nested_batches_of_inference_input( inference_input=image, ) - inject_images_into_payload( + inject_nested_batches_of_images_into_payload( payload=inputs, encoded_images=loaded_image, key=image_name, diff --git a/inference_sdk/http/utils/loaders.py b/inference_sdk/http/utils/loaders.py index 9e398803f..24721da2a 100644 --- a/inference_sdk/http/utils/loaders.py +++ b/inference_sdk/http/utils/loaders.py @@ -52,6 +52,29 @@ def load_directory_inference_input( yield path, cv2.imread(path) +def load_nested_batches_of_inference_input( + inference_input: Union[list, ImagesReference], + max_height: Optional[int] = None, + max_width: Optional[int] = None, +) -> Union[Tuple[str, Optional[float]], list]: + if not isinstance(inference_input, list): + return load_static_inference_input( + inference_input=inference_input, + max_height=max_height, + max_width=max_width, + )[0] + result = [] + for element in inference_input: + result.append( + load_nested_batches_of_inference_input( + inference_input=element, + max_height=max_height, + max_width=max_width, + ) + ) + return result + + def load_static_inference_input( inference_input: Union[ImagesReference, List[ImagesReference]], max_height: Optional[int] = None, diff --git a/inference_sdk/http/utils/requests.py b/inference_sdk/http/utils/requests.py index b38b1f9e5..e2b607f96 100644 --- a/inference_sdk/http/utils/requests.py +++ b/inference_sdk/http/utils/requests.py @@ -1,5 +1,5 @@ import re -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Union from requests import Response @@ -44,3 +44,30 @@ def inject_images_into_payload( else: payload[key] = {"type": "base64", "value": encoded_images[0][0]} return payload + + +def inject_nested_batches_of_images_into_payload( + payload: dict, + encoded_images: Union[list, Tuple[str, Optional[float]]], + key: str = "image", +) -> dict: + payload_value = _batch_of_images_into_inference_format( + encoded_images=encoded_images, + ) + payload[key] = payload_value + return payload + + +def _batch_of_images_into_inference_format( + encoded_images: Union[list, Tuple[str, Optional[float]]], +) -> Union[dict, list]: + if not isinstance(encoded_images, list): + return {"type": "base64", "value": encoded_images[0]} + result = [] + for element in encoded_images: + result.append( + _batch_of_images_into_inference_format( + encoded_images=element, + ) + ) + return result diff --git a/requirements/_requirements.txt b/requirements/_requirements.txt index 1d65cb8ad..dfe894b91 100644 --- a/requirements/_requirements.txt +++ b/requirements/_requirements.txt @@ -1,34 +1,35 @@ -aiortc>=1.9.0 -APScheduler<=3.10.1 -cython<=3.0.0 -python-dotenv<=2.0.0 +aiortc>=1.9.0,<2.0.0 +APScheduler>=3.10.1,<4.0.0 +cython~=3.0.0 +python-dotenv~=1.0.0 fastapi>=0.100,<0.111 numpy<=1.26.4 opencv-python>=4.8.1.78,<=4.10.0.84 -piexif<=1.1.3 +piexif~=1.1.3 pillow<11.0 prometheus-fastapi-instrumentator<=6.0.0 -redis<6.0.0 -requests>=2.26.0 -rich<=13.5.2 +redis~=5.0.0 +requests>=2.32.0,<3.0.0 +rich~=13.0.0 supervision>=0.21.0,<=0.22.0 -pybase64<2.0.0 -scikit-image>=0.19.0 -requests-toolbelt>=1.0.0 -wheel>=0.38.1 -setuptools>=70.0.0,<=72.1.0 -pytest-asyncio<=0.21.1 -networkx>=3.1 +pybase64~=1.0.0 +scikit-image>=0.19.0,<=0.24.0 +requests-toolbelt~=1.0.0 +wheel>=0.38.1,<=0.45.0 +setuptools>=70.0.0 # lack of upper-bound to ensure compatibility with Google Colab (builds to define one if needed) +networkx~=3.1 pydantic~=2.6 pydantic-settings~=2.2 -openai>=1.12.0 -structlog>=24.1.0 -zxing-cpp>=2.2.0 -boto3<=1.34.123 -typing_extensions>=4.8.0 -pydot>=2.0.0 +openai>=1.12.0,<2.0.0 +structlog>=24.1.0,<25.0.0 +zxing-cpp~=2.2.0 +boto3<=1.35.60 +typing_extensions>=4.8.0,<=4.12.2 +pydot~=2.0.0 shapely>=2.0.0,<2.1.0 tldextract~=5.1.2 packaging~=24.0 anthropic~=0.34.2 pandas>=2.0.0,<2.3.0 +pytest>=8.0.0,<9.0.0 # this is not a joke, sam2 requires this as the fork we are using is dependent on that, yet +# do not mark the dependency: https://github.com/SauravMaheshkar/samv2/blob/main/sam2/utils/download.py \ No newline at end of file diff --git a/requirements/requirements.cli.txt b/requirements/requirements.cli.txt index a703e8e50..16e5272a7 100644 --- a/requirements/requirements.cli.txt +++ b/requirements/requirements.cli.txt @@ -1,12 +1,12 @@ -requests<=2.31.0 -docker==6.1.3 +requests>=2.32.0,<3.0.0 +docker>=7.0.0,<8.0.0 typer>=0.9.0,<=0.12.5 -rich<=13.5.2 -PyYAML>=6.0.0 -supervision>=0.20.0,<1.0.0 +rich~=13.0.0 +PyYAML~=6.0.0 +supervision>=0.21.0,<=0.22.0 opencv-python>=4.8.1.78,<=4.10.0.84 -tqdm>=4.0.0 -GPUtil>=1.4.0 -py-cpuinfo>=9.0.0 -aiohttp>=3.9.0 -backoff>=2.2.0 +tqdm>=4.0.0,<5.0.0 +GPUtil~=1.4.0 +py-cpuinfo~=9.0.0 +aiohttp>=3.9.0,<=3.10.11 +backoff~=2.2.0 diff --git a/requirements/requirements.doctr.txt b/requirements/requirements.doctr.txt index c361bf091..bd1d66c37 100644 --- a/requirements/requirements.doctr.txt +++ b/requirements/requirements.doctr.txt @@ -1,2 +1,2 @@ -python-doctr[torch] -tf2onnx \ No newline at end of file +python-doctr[torch]>=0.7.0,<=0.10.0 +tf2onnx~=1.16.0 \ No newline at end of file diff --git a/requirements/requirements.hosted.txt b/requirements/requirements.hosted.txt index 821cbac7b..da447936e 100644 --- a/requirements/requirements.hosted.txt +++ b/requirements/requirements.hosted.txt @@ -1,3 +1,2 @@ -pymemcache<=4.0.0 -elasticache_auto_discovery<=1.0.0 -prometheus-fastapi-instrumentator<=6.0.0 \ No newline at end of file +pymemcache~=4.0.0 +elasticache_auto_discovery~=1.0.0 diff --git a/requirements/requirements.http.txt b/requirements/requirements.http.txt index 04e6b7132..d6249c883 100644 --- a/requirements/requirements.http.txt +++ b/requirements/requirements.http.txt @@ -1,5 +1,5 @@ uvicorn[standard]<=0.22.0 python-multipart>=0.0.7,<=0.0.9 fastapi-cprofile<=0.0.2 -orjson>=3.9.10 -asgi_correlation_id>=4.3.1 +orjson>=3.9.10,<=3.10.11 +asgi_correlation_id~=4.3.1 diff --git a/requirements/requirements.jetson.txt b/requirements/requirements.jetson.txt index dbcbeed61..c67662ca5 100644 --- a/requirements/requirements.jetson.txt +++ b/requirements/requirements.jetson.txt @@ -1,4 +1,4 @@ -pypdfium2 -jupyterlab -PyYAML -onnxruntime-gpu +pypdfium2~=4.0.0 +jupyterlab>=4.3.0,<5.0.0 +PyYAML~=6.0.0 +onnxruntime-gpu>=1.15.1,<1.20.0 diff --git a/requirements/requirements.parallel.txt b/requirements/requirements.parallel.txt index d330682d7..6c576396f 100644 --- a/requirements/requirements.parallel.txt +++ b/requirements/requirements.parallel.txt @@ -1,2 +1,2 @@ -celery -gunicorn \ No newline at end of file +celery>=5.4.0,<6.0.0 +gunicorn~=23.0.0 \ No newline at end of file diff --git a/requirements/requirements.sdk.http.txt b/requirements/requirements.sdk.http.txt index 6d86bb690..74f4e735e 100644 --- a/requirements/requirements.sdk.http.txt +++ b/requirements/requirements.sdk.http.txt @@ -1,11 +1,9 @@ -requests>=2.0.0 -dataclasses-json>=0.6.0 +requests>=2.32.0,<3.0.0 +dataclasses-json~=0.6.0 opencv-python>=4.8.1.78,<=4.10.0.84 -pillow>=9.0.0 -requests>=2.27.0 -supervision>=0.20.0,<1.0.0 +pillow>=9.0.0,<11.0 +supervision>=0.21.0,<=0.22.0 numpy<=1.26.4 -aiohttp>=3.9.0 -backoff>=2.2.0 -aioresponses>=0.7.6 -py-cpuinfo>=9.0.0 +aiohttp>=3.9.0,<=3.10.11 +backoff~=2.2.0 +py-cpuinfo~=9.0.0 diff --git a/requirements/requirements.test.unit.txt b/requirements/requirements.test.unit.txt index c45356659..a403444b2 100644 --- a/requirements/requirements.test.unit.txt +++ b/requirements/requirements.test.unit.txt @@ -10,4 +10,4 @@ pytest-timeout>=2.2.0 httpx uvicorn<=0.22.0 aioresponses>=0.7.6 -supervision>=0.20.0,<1.0.0 \ No newline at end of file +supervision>=0.21.0,<=0.22.0 \ No newline at end of file diff --git a/requirements/requirements.waf.txt b/requirements/requirements.waf.txt index e4f807a3d..d5b5e0631 100644 --- a/requirements/requirements.waf.txt +++ b/requirements/requirements.waf.txt @@ -1 +1 @@ -metlo \ No newline at end of file +metlo>=0.0.17,<=0.1.5 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 0c40096e3..66fcdd0ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ import os os.environ["TELEMETRY_OPT_OUT"] = "True" +os.environ["ONNXRUNTIME_EXECUTION_PROVIDERS"] = "[CPUExecutionProvider]" diff --git a/tests/inference/hosted_platform_tests/test_workflows.py b/tests/inference/hosted_platform_tests/test_workflows.py index bbe4c57a1..b3ff50620 100644 --- a/tests/inference/hosted_platform_tests/test_workflows.py +++ b/tests/inference/hosted_platform_tests/test_workflows.py @@ -129,7 +129,7 @@ def test_get_versions_of_execution_engine(object_detection_service_url: str) -> # then response.raise_for_status() response_data = response.json() - assert response_data["versions"] == ["1.2.0"] + assert response_data["versions"] == ["1.3.0"] FUNCTION = """ diff --git a/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_claude.py b/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_claude.py index 420b213cd..ef43d7bd9 100644 --- a/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_claude.py +++ b/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_claude.py @@ -24,7 +24,7 @@ "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$inputs.image", "vlm_output": "$steps.claude.output", @@ -183,7 +183,7 @@ def test_structured_parsing_workflow( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_detector@v1", + "type": "roboflow_core/vlm_as_detector@v2", "name": "parser", "vlm_output": "$steps.claude.output", "image": "$inputs.image", @@ -281,7 +281,7 @@ def test_object_detection_workflow( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$steps.cropping.crops", "vlm_output": "$steps.claude.output", diff --git a/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_gemini.py b/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_gemini.py index 5fb90aa37..c61344c00 100644 --- a/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_gemini.py +++ b/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_gemini.py @@ -24,7 +24,7 @@ "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$inputs.image", "vlm_output": "$steps.gemini.output", @@ -183,7 +183,7 @@ def test_structured_parsing_workflow( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_detector@v1", + "type": "roboflow_core/vlm_as_detector@v2", "name": "parser", "vlm_output": "$steps.gemini.output", "image": "$inputs.image", @@ -281,7 +281,7 @@ def test_object_detection_workflow( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$steps.cropping.crops", "vlm_output": "$steps.gemini.output", diff --git a/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_openai.py b/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_openai.py index edd4d5137..112d4342e 100644 --- a/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_openai.py +++ b/tests/inference/hosted_platform_tests/workflows_examples/test_workflow_with_openai.py @@ -114,7 +114,7 @@ def test_image_description_workflow( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$inputs.image", "vlm_output": "$steps.gpt.output", @@ -294,7 +294,7 @@ def test_structured_prompting_workflow( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$steps.cropping.crops", "vlm_output": "$steps.gpt.output", diff --git a/tests/inference/integration_tests/test_workflow_endpoints.py b/tests/inference/integration_tests/test_workflow_endpoints.py index c7c38cb20..f3ca64a1b 100644 --- a/tests/inference/integration_tests/test_workflow_endpoints.py +++ b/tests/inference/integration_tests/test_workflow_endpoints.py @@ -691,7 +691,7 @@ def test_get_versions_of_execution_engine(server_url: str) -> None: # then response.raise_for_status() response_data = response.json() - assert response_data["versions"] == ["1.2.0"] + assert response_data["versions"] == ["1.3.0"] def test_getting_block_schema_using_get_endpoint(server_url) -> None: diff --git a/tests/inference/unit_tests/core/cache/test_serializers.py b/tests/inference/unit_tests/core/cache/test_serializers.py index 8c982f6de..0d294e31b 100644 --- a/tests/inference/unit_tests/core/cache/test_serializers.py +++ b/tests/inference/unit_tests/core/cache/test_serializers.py @@ -1,9 +1,11 @@ import os from unittest.mock import MagicMock + import pytest + from inference.core.cache.serializers import ( - to_cachable_inference_item, build_condensed_response, + to_cachable_inference_item, ) from inference.core.entities.requests.inference import ( ClassificationInferenceRequest, @@ -11,16 +13,16 @@ ) from inference.core.entities.responses.inference import ( ClassificationInferenceResponse, - MultiLabelClassificationInferenceResponse, + ClassificationPrediction, InstanceSegmentationInferenceResponse, + InstanceSegmentationPrediction, + Keypoint, KeypointsDetectionInferenceResponse, + KeypointsPrediction, + MultiLabelClassificationInferenceResponse, + MultiLabelClassificationPrediction, ObjectDetectionInferenceResponse, ObjectDetectionPrediction, - ClassificationPrediction, - MultiLabelClassificationPrediction, - InstanceSegmentationPrediction, - KeypointsPrediction, - Keypoint, Point, ) diff --git a/tests/inference/unit_tests/core/interfaces/http/handlers/__init__.py b/tests/inference/unit_tests/core/interfaces/http/handlers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/inference/unit_tests/core/interfaces/http/handlers/test_workflows.py b/tests/inference/unit_tests/core/interfaces/http/handlers/test_workflows.py new file mode 100644 index 000000000..181a56ac3 --- /dev/null +++ b/tests/inference/unit_tests/core/interfaces/http/handlers/test_workflows.py @@ -0,0 +1,85 @@ +from inference.core.interfaces.http.handlers.workflows import ( + filter_out_unwanted_workflow_outputs, +) + + +def test_filter_out_unwanted_workflow_outputs_when_nothing_to_filter() -> None: + # given + workflow_results = [ + {"a": 1, "b": 2}, + {"a": 3, "b": 4}, + ] + + # when + result = filter_out_unwanted_workflow_outputs( + workflow_results=workflow_results, + excluded_fields=None, + ) + + # then + assert result == [ + {"a": 1, "b": 2}, + {"a": 3, "b": 4}, + ] + + +def test_filter_out_unwanted_workflow_outputs_when_empty_filter() -> None: + # given + workflow_results = [ + {"a": 1, "b": 2}, + {"a": 3, "b": 4}, + ] + + # when + result = filter_out_unwanted_workflow_outputs( + workflow_results=workflow_results, + excluded_fields=[], + ) + + # then + assert result == [ + {"a": 1, "b": 2}, + {"a": 3, "b": 4}, + ] + + +def test_filter_out_unwanted_workflow_outputs_when_fields_to_be_filtered() -> None: + # given + workflow_results = [ + {"a": 1, "b": 2}, + {"a": 3, "b": 4}, + ] + + # when + result = filter_out_unwanted_workflow_outputs( + workflow_results=workflow_results, + excluded_fields=["a"], + ) + + # then + assert result == [ + {"b": 2}, + {"b": 4}, + ] + + +def test_filter_out_unwanted_workflow_outputs_when_filter_defines_non_existing_fields() -> ( + None +): + # given + workflow_results = [ + {"a": 1, "b": 2}, + {"a": 3, "b": 4}, + ] + + # when + result = filter_out_unwanted_workflow_outputs( + workflow_results=workflow_results, + excluded_fields=["non-existing"], + ) + + # then + assert result == [ + {"a": 1, "b": 2}, + {"a": 3, "b": 4}, + ] diff --git a/tests/inference/unit_tests/core/interfaces/http/test_orjson_utils.py b/tests/inference/unit_tests/core/interfaces/http/test_orjson_utils.py index 4572129f2..73f5f10d6 100644 --- a/tests/inference/unit_tests/core/interfaces/http/test_orjson_utils.py +++ b/tests/inference/unit_tests/core/interfaces/http/test_orjson_utils.py @@ -1,50 +1,12 @@ -import base64 - -import cv2 import numpy as np -from inference.core.interfaces.http.orjson_utils import ( - serialise_list, - serialise_workflow_result, -) +from inference.core.interfaces.http.orjson_utils import serialise_workflow_result from inference.core.workflows.execution_engine.entities.base import ( ImageParentMetadata, WorkflowImageData, ) -def test_serialise_list() -> None: - # given - np_image = np.zeros((192, 168, 3), dtype=np.uint8) - elements = [ - 3, - "some", - WorkflowImageData( - parent_metadata=ImageParentMetadata(parent_id="some"), - numpy_image=np_image, - ), - ] - - # when - result = serialise_list(elements=elements) - - # then - assert len(result) == 3, "The same number of elements must be returned" - assert result[0] == 3, "First element of list must be untouched" - assert result[1] == "some", "Second element of list must be untouched" - assert ( - result[2]["type"] == "base64" - ), "Type of third element must be changed into base64" - decoded = base64.b64decode(result[2]["value"]) - recovered_image = cv2.imdecode( - np.fromstring(decoded, dtype=np.uint8), - cv2.IMREAD_UNCHANGED, - ) - assert ( - recovered_image == np_image - ).all(), "Recovered image should be equal to input image" - - def test_serialise_workflow_result() -> None: # given np_image = np.zeros((192, 168, 3), dtype=np.uint8) diff --git a/tests/inference/unit_tests/usage_tracking/test_collector.py b/tests/inference/unit_tests/usage_tracking/test_collector.py index 96f7aaa7f..a25553c54 100644 --- a/tests/inference/unit_tests/usage_tracking/test_collector.py +++ b/tests/inference/unit_tests/usage_tracking/test_collector.py @@ -764,7 +764,7 @@ def test_zip_usage_payloads_with_different_exec_session_ids(): "fps": 10, "exec_session_id": "session_2", }, - } + }, }, { "fake_api1_hash": { @@ -831,7 +831,11 @@ def test_zip_usage_payloads_with_different_exec_session_ids(): def test_system_info_with_dedicated_deployment_id(): # given - system_info = UsageCollector.system_info(ip_address="w.x.y.z", hostname="hostname01", dedicated_deployment_id="deployment01") + system_info = UsageCollector.system_info( + ip_address="w.x.y.z", + hostname="hostname01", + dedicated_deployment_id="deployment01", + ) # then expected_system_info = { @@ -845,7 +849,9 @@ def test_system_info_with_dedicated_deployment_id(): def test_system_info_with_no_dedicated_deployment_id(): # given - system_info = UsageCollector.system_info(ip_address="w.x.y.z", hostname="hostname01") + system_info = UsageCollector.system_info( + ip_address="w.x.y.z", hostname="hostname01" + ) # then expected_system_info = { @@ -855,3 +861,31 @@ def test_system_info_with_no_dedicated_deployment_id(): } for k, v in expected_system_info.items(): assert system_info[k] == v + + +def test_record_malformed_usage(): + # given + collector = UsageCollector() + + # when + collector.record_usage( + source=None, + category="model", + frames=None, + api_key="fake", + resource_details=None, + resource_id=None, + inference_test_run=None, + fps=None, + ) + + # then + assert "fake" in collector._usage + assert "model:None" in collector._usage["fake"] + assert collector._usage["fake"]["model:None"]["processed_frames"] == 0 + assert collector._usage["fake"]["model:None"]["fps"] == 0 + assert collector._usage["fake"]["model:None"]["source_duration"] == 0 + assert collector._usage["fake"]["model:None"]["category"] == "model" + assert collector._usage["fake"]["model:None"]["resource_id"] == None + assert collector._usage["fake"]["model:None"]["resource_details"] == "{}" + assert collector._usage["fake"]["model:None"]["api_key_hash"] == "fake" diff --git a/tests/inference_sdk/unit_tests/http/test_client.py b/tests/inference_sdk/unit_tests/http/test_client.py index a92fe07da..bbbbbc347 100644 --- a/tests/inference_sdk/unit_tests/http/test_client.py +++ b/tests/inference_sdk/unit_tests/http/test_client.py @@ -3575,7 +3575,7 @@ def test_infer_from_workflow_when_no_parameters_given( }, "Request payload must contain api key and inputs" -@mock.patch.object(client, "load_static_inference_input") +@mock.patch.object(client, "load_nested_batches_of_inference_input") @pytest.mark.parametrize( "legacy_endpoints, endpoint_to_use, parameter_name", [ @@ -3584,7 +3584,7 @@ def test_infer_from_workflow_when_no_parameters_given( ], ) def test_infer_from_workflow_when_parameters_and_excluded_fields_given( - load_static_inference_input_mock: MagicMock, + load_nested_batches_of_inference_input_mock: MagicMock, requests_mock: Mocker, legacy_endpoints: bool, endpoint_to_use: str, @@ -3599,8 +3599,8 @@ def test_infer_from_workflow_when_parameters_and_excluded_fields_given( "outputs": [{"some": 3}], }, ) - load_static_inference_input_mock.side_effect = [ - [("base64_image_1", 0.5)], + load_nested_batches_of_inference_input_mock.side_effect = [ + ("base64_image_1", 0.5), [("base64_image_2", 0.5), ("base64_image_3", 0.5)], ] method = ( @@ -3647,7 +3647,7 @@ def test_infer_from_workflow_when_parameters_and_excluded_fields_given( }, "Request payload must contain api key and inputs" -@mock.patch.object(client, "load_static_inference_input") +@mock.patch.object(client, "load_nested_batches_of_inference_input") @pytest.mark.parametrize( "legacy_endpoints, endpoint_to_use, parameter_name", [ @@ -3656,7 +3656,7 @@ def test_infer_from_workflow_when_parameters_and_excluded_fields_given( ], ) def test_infer_from_workflow_when_usage_of_cache_disabled( - load_static_inference_input_mock: MagicMock, + load_nested_batches_of_inference_input_mock: MagicMock, requests_mock: Mocker, legacy_endpoints: bool, endpoint_to_use: str, @@ -3671,8 +3671,8 @@ def test_infer_from_workflow_when_usage_of_cache_disabled( "outputs": [{"some": 3}], }, ) - load_static_inference_input_mock.side_effect = [ - [("base64_image_1", 0.5)], + load_nested_batches_of_inference_input_mock.side_effect = [ + ("base64_image_1", 0.5), [("base64_image_2", 0.5), ("base64_image_3", 0.5)], ] method = ( @@ -3714,7 +3714,7 @@ def test_infer_from_workflow_when_usage_of_cache_disabled( }, "Request payload must contain api key, inputs and no cache flag" -@mock.patch.object(client, "load_static_inference_input") +@mock.patch.object(client, "load_nested_batches_of_inference_input") @pytest.mark.parametrize( "legacy_endpoints, endpoint_to_use, parameter_name", [ @@ -3723,7 +3723,7 @@ def test_infer_from_workflow_when_usage_of_cache_disabled( ], ) def test_infer_from_workflow_when_usage_of_profiler_enabled( - load_static_inference_input_mock: MagicMock, + load_nested_batches_of_inference_input_mock: MagicMock, requests_mock: Mocker, legacy_endpoints: bool, endpoint_to_use: str, @@ -3742,8 +3742,8 @@ def test_infer_from_workflow_when_usage_of_profiler_enabled( "profiler_trace": [{"my": "trace"}] }, ) - load_static_inference_input_mock.side_effect = [ - [("base64_image_1", 0.5)], + load_nested_batches_of_inference_input_mock.side_effect = [ + ("base64_image_1", 0.5), [("base64_image_2", 0.5), ("base64_image_3", 0.5)], ] method = ( @@ -3790,6 +3790,87 @@ def test_infer_from_workflow_when_usage_of_profiler_enabled( assert data == [{"my": "trace"}], "Trace content must be fully saved" +@mock.patch.object(client, "load_nested_batches_of_inference_input") +@pytest.mark.parametrize( + "legacy_endpoints, endpoint_to_use, parameter_name", + [ + (True, "/infer/workflows/my_workspace/my_workflow", "workflow_name"), + (False, "/my_workspace/workflows/my_workflow", "workflow_id"), + ], +) +def test_infer_from_workflow_when_nested_batch_of_inputs_provided( + load_nested_batches_of_inference_input_mock: MagicMock, + requests_mock: Mocker, + legacy_endpoints: bool, + endpoint_to_use: str, + parameter_name: str, +) -> None: + # given + api_url = "http://some.com" + http_client = InferenceHTTPClient(api_key="my-api-key", api_url=api_url) + requests_mock.post( + f"{api_url}{endpoint_to_use}", + json={ + "outputs": [{"some": 3}], + }, + ) + load_nested_batches_of_inference_input_mock.side_effect = [ + [ + [("base64_image_1", 0.5), ("base64_image_2", 0.5)], + [("base64_image_3", 0.5), ("base64_image_4", 0.5), ("base64_image_5", 0.5)], + [("base64_image_6", 0.5)], + ], + ] + method = ( + http_client.infer_from_workflow + if legacy_endpoints + else http_client.run_workflow + ) + + # when + result = method( + workspace_name="my_workspace", + images={"image_1": [["1", "2"], ["3", "4", "5"], ["6"]]}, + parameters={ + "batch_oriented_param": [ + ["a", "b"], + ["c", "d", "e"], + ["f"] + ] + }, + **{parameter_name: "my_workflow"}, + ) + + # then + assert result == [{"some": 3}], "Response from API must be properly decoded" + assert requests_mock.request_history[0].json() == { + "api_key": "my-api-key", + "use_cache": True, + "enable_profiling": False, + "inputs": { + "image_1": [ + [ + {"type": "base64", "value": "base64_image_1"}, + {"type": "base64", "value": "base64_image_2"}, + ], + [ + {"type": "base64", "value": "base64_image_3"}, + {"type": "base64", "value": "base64_image_4"}, + {"type": "base64", "value": "base64_image_5"}, + ], + [ + {"type": "base64", "value": "base64_image_6"}, + ], + ], + "batch_oriented_param": [ + ["a", "b"], + ["c", "d", "e"], + ["f"], + ], + }, + }, "Request payload must contain api key, inputs and no cache flag" + + @pytest.mark.parametrize( "legacy_endpoints, endpoint_to_use, parameter_name", [ @@ -3849,13 +3930,13 @@ def test_infer_from_workflow_when_both_workflow_name_and_specs_given() -> None: ) -@mock.patch.object(client, "load_static_inference_input") +@mock.patch.object(client, "load_nested_batches_of_inference_input") @pytest.mark.parametrize( "legacy_endpoints, endpoint_to_use", [(True, "/infer/workflows"), (False, "/workflows/run")], ) def test_infer_from_workflow_when_custom_workflow_with_both_parameters_and_excluded_fields_given( - load_static_inference_input_mock: MagicMock, + load_nested_batches_of_inference_input_mock: MagicMock, requests_mock: Mocker, legacy_endpoints: bool, endpoint_to_use: str, @@ -3869,8 +3950,8 @@ def test_infer_from_workflow_when_custom_workflow_with_both_parameters_and_exclu "outputs": [{"some": 3}], }, ) - load_static_inference_input_mock.side_effect = [ - [("base64_image_1", 0.5)], + load_nested_batches_of_inference_input_mock.side_effect = [ + ("base64_image_1", 0.5), [("base64_image_2", 0.5), ("base64_image_3", 0.5)], ] method = ( diff --git a/tests/inference_sdk/unit_tests/http/utils/test_loaders.py b/tests/inference_sdk/unit_tests/http/utils/test_loaders.py index 2d83ec9c0..55dc9d21d 100644 --- a/tests/inference_sdk/unit_tests/http/utils/test_loaders.py +++ b/tests/inference_sdk/unit_tests/http/utils/test_loaders.py @@ -22,7 +22,7 @@ load_static_inference_input, load_static_inference_input_async, load_stream_inference_input, - uri_is_http_link, + uri_is_http_link, load_nested_batches_of_inference_input, ) @@ -650,3 +650,63 @@ def test_load_stream_inference_input( get_video_frames_generator_mock.assert_called_once_with( source_path="/some/video.mp4" ) + + +@mock.patch.object(loaders, "load_static_inference_input") +def test_load_nested_batches_of_inference_input_when_single_element_is_given( + load_static_inference_input_mock: MagicMock, +) -> None: + # given + load_static_inference_input_mock.side_effect = [ + ["image_1"] + ] + + # when + result = load_nested_batches_of_inference_input( + inference_input="my_image", + ) + + # then + assert result == "image_1", "Expected direct result from load_static_inference_input()" + + +@mock.patch.object(loaders, "load_static_inference_input") +def test_load_nested_batches_of_inference_input_when_1d_batch_is_given( + load_static_inference_input_mock: MagicMock, +) -> None: + # given + load_static_inference_input_mock.side_effect = [ + ["image_1"], + ["image_2"], + ["image_3"] + ] + + # when + result = load_nested_batches_of_inference_input( + inference_input=["1", "2", "3"], + ) + + # then + assert result == ["image_1", "image_2", "image_3"], "Expected direct result from load_static_inference_input()" + + +@mock.patch.object(loaders, "load_static_inference_input") +def test_load_nested_batches_of_inference_input_when_nested_batch_is_given( + load_static_inference_input_mock: MagicMock, +) -> None: + # given + load_static_inference_input_mock.side_effect = [ + ["image_1"], + ["image_2"], + ["image_3"], + ["image_4"], + ["image_5"], + ] + + # when + result = load_nested_batches_of_inference_input( + inference_input=[["1", "2"], ["3"], [["4", "5"]]], + ) + + # then + assert result == [["image_1", "image_2"], ["image_3"], [["image_4", "image_5"]]] diff --git a/tests/inference_sdk/unit_tests/http/utils/test_requests.py b/tests/inference_sdk/unit_tests/http/utils/test_requests.py index b4d131895..59ff545be 100644 --- a/tests/inference_sdk/unit_tests/http/utils/test_requests.py +++ b/tests/inference_sdk/unit_tests/http/utils/test_requests.py @@ -5,7 +5,7 @@ API_KEY_PATTERN, api_key_safe_raise_for_status, deduct_api_key, - inject_images_into_payload, + inject_images_into_payload, inject_nested_batches_of_images_into_payload, ) @@ -146,3 +146,49 @@ def test_inject_images_into_payload_when_payload_key_is_specified() -> None: "my": "payload", "prompt": {"type": "base64", "value": "image_payload_1"}, }, "Payload is expected to be extended with the content of only single image under `prompt` key" + + +def test_inject_nested_batches_of_images_into_payload_when_single_image_given() -> None: + # when + result = inject_nested_batches_of_images_into_payload( + payload={}, + encoded_images=("img1", None), + ) + + # then + assert result == {"image": {"type": "base64", "value": "img1"}} + + +def test_inject_nested_batches_of_images_into_payload_when_1d_batch_of_images_given() -> None: + # when + result = inject_nested_batches_of_images_into_payload( + payload={}, + encoded_images=[("img1", None), ("img2", None)], + ) + + # then + assert result == { + "image": [ + {"type": "base64", "value": "img1"}, + {"type": "base64", "value": "img2"}, + ] + } + + +def test_inject_nested_batches_of_images_into_payload_when_nested_batch_of_images_given() -> None: + # when + result = inject_nested_batches_of_images_into_payload( + payload={}, + encoded_images=[[("img1", None)], [("img2", None), ("img3", None)]], + ) + + # then + assert result == { + "image": [ + [{"type": "base64", "value": "img1"}], + [ + {"type": "base64", "value": "img2"}, + {"type": "base64", "value": "img3"}, + ], + ] + } diff --git a/tests/workflows/integration_tests/execution/stub_plugins/dimensionality_manipulation_plugin/tile_detections_batch.py b/tests/workflows/integration_tests/execution/stub_plugins/dimensionality_manipulation_plugin/tile_detections_batch.py index eeb4d5da6..d00cf2168 100644 --- a/tests/workflows/integration_tests/execution/stub_plugins/dimensionality_manipulation_plugin/tile_detections_batch.py +++ b/tests/workflows/integration_tests/execution/stub_plugins/dimensionality_manipulation_plugin/tile_detections_batch.py @@ -82,7 +82,7 @@ def run( images_crops: Batch[Batch[WorkflowImageData]], crops_predictions: Batch[Batch[sv.Detections]], ) -> BlockResult: - annotator = sv.BoundingBoxAnnotator() + annotator = sv.BoxAnnotator() visualisations = [] for image_crops, crop_predictions in zip(images_crops, crops_predictions): visualisations_batch_element = [] diff --git a/tests/workflows/integration_tests/execution/stub_plugins/dimensionality_manipulation_plugin/tile_detections_non_batch.py b/tests/workflows/integration_tests/execution/stub_plugins/dimensionality_manipulation_plugin/tile_detections_non_batch.py index cca572d1e..ab51455d2 100644 --- a/tests/workflows/integration_tests/execution/stub_plugins/dimensionality_manipulation_plugin/tile_detections_non_batch.py +++ b/tests/workflows/integration_tests/execution/stub_plugins/dimensionality_manipulation_plugin/tile_detections_non_batch.py @@ -78,7 +78,7 @@ def run( crops: Batch[WorkflowImageData], crops_predictions: Batch[sv.Detections], ) -> BlockResult: - annotator = sv.BoundingBoxAnnotator() + annotator = sv.BoxAnnotator() visualisations = [] for image, prediction in zip(crops, crops_predictions): annotated_image = annotator.annotate( diff --git a/tests/workflows/integration_tests/execution/stub_plugins/mixed_input_characteristic_plugin/__init__.py b/tests/workflows/integration_tests/execution/stub_plugins/mixed_input_characteristic_plugin/__init__.py new file mode 100644 index 000000000..0a23fa7eb --- /dev/null +++ b/tests/workflows/integration_tests/execution/stub_plugins/mixed_input_characteristic_plugin/__init__.py @@ -0,0 +1,354 @@ +from typing import Any, Dict, List, Literal, Type, Union + +from pydantic import ConfigDict + +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, +) +from inference.core.workflows.execution_engine.entities.types import ( + FLOAT_ZERO_TO_ONE_KIND, + Selector, + WorkflowParameterSelector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) + + +class NonBatchInputBlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "short_description": "", + "long_description": "", + "license": "Apache-2.0", + "block_type": "dummy", + } + ) + type: Literal["NonBatchInputBlock"] + non_batch_parameter: Union[WorkflowParameterSelector(), Any] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="float_value", + kind=[FLOAT_ZERO_TO_ONE_KIND], + ), + ] + + +class NonBatchInputBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return NonBatchInputBlockManifest + + def run(self, non_batch_parameter: Any) -> BlockResult: + return {"float_value": 0.4} + + +class MixedInputWithoutBatchesBlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "short_description": "", + "long_description": "", + "license": "Apache-2.0", + "block_type": "dummy", + } + ) + type: Literal["MixedInputWithoutBatchesBlock"] + mixed_parameter: Union[ + Selector(), + Any, + ] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="float_value", + kind=[FLOAT_ZERO_TO_ONE_KIND], + ), + ] + + +class MixedInputWithoutBatchesBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return MixedInputWithoutBatchesBlockManifest + + def run(self, mixed_parameter: Any) -> BlockResult: + return {"float_value": 0.4} + + +class MixedInputWithBatchesBlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "short_description": "", + "long_description": "", + "license": "Apache-2.0", + "block_type": "dummy", + } + ) + type: Literal["MixedInputWithBatchesBlock"] + mixed_parameter: Union[ + Selector(), + Any, + ] + + @classmethod + def get_parameters_accepting_batches_and_scalars(cls) -> List[str]: + return ["mixed_parameter"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="float_value", + kind=[FLOAT_ZERO_TO_ONE_KIND], + ), + ] + + +class MixedInputWithBatchesBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return MixedInputWithBatchesBlockManifest + + def run(self, mixed_parameter: Union[Batch[Any], Any]) -> BlockResult: + if isinstance(mixed_parameter, Batch): + return [{"float_value": 0.4}] * len(mixed_parameter) + return {"float_value": 0.4} + + +class BatchInputBlockProcessingBatchesManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "short_description": "", + "long_description": "", + "license": "Apache-2.0", + "block_type": "dummy", + } + ) + type: Literal["BatchInputBlockProcessingBatches"] + batch_parameter: Selector() + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["batch_parameter"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="float_value", + kind=[FLOAT_ZERO_TO_ONE_KIND], + ), + ] + + +class BatchInputProcessingBatchesBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BatchInputBlockProcessingBatchesManifest + + def run(self, batch_parameter: Batch[Any]) -> BlockResult: + if not isinstance(batch_parameter, Batch): + raise ValueError("Batch[X] must be provided") + return [{"float_value": 0.4}] * len(batch_parameter) + + +class BatchInputBlockProcessingNotBatchesManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "short_description": "", + "long_description": "", + "license": "Apache-2.0", + "block_type": "dummy", + } + ) + type: Literal["BatchInputBlockNotProcessingBatches"] + batch_parameter: Selector() + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="float_value", + kind=[FLOAT_ZERO_TO_ONE_KIND], + ), + ] + + +class BatchInputNotProcessingBatchesBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BatchInputBlockProcessingNotBatchesManifest + + def run(self, batch_parameter: Any) -> BlockResult: + return {"float_value": 0.4} + + +class CompoundNonBatchInputBlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "short_description": "", + "long_description": "", + "license": "Apache-2.0", + "block_type": "dummy", + } + ) + type: Literal["CompoundNonBatchInputBlock"] + compound_parameter: Dict[str, Union[Selector(), Any]] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="float_value", + kind=[FLOAT_ZERO_TO_ONE_KIND], + ), + ] + + +class CompoundNonBatchInputBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return CompoundNonBatchInputBlockManifest + + def run(self, compound_parameter: Dict[str, Any]) -> BlockResult: + return {"float_value": 0.4} + + +class CompoundMixedInputBlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "short_description": "", + "long_description": "", + "license": "Apache-2.0", + "block_type": "dummy", + } + ) + type: Literal["CompoundMixedInputBlockManifestBlock"] + compound_parameter: Dict[str, Union[Selector(), Any]] + + @classmethod + def get_parameters_accepting_batches_and_scalars(cls) -> List[str]: + return ["compound_parameter"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="float_value", + kind=[FLOAT_ZERO_TO_ONE_KIND], + ), + ] + + +class CompoundMixedInputBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return CompoundMixedInputBlockManifest + + def run(self, compound_parameter: Dict[str, Any]) -> BlockResult: + retrieved_batches = [ + v for v in compound_parameter.values() if isinstance(v, Batch) + ] + if not retrieved_batches: + return {"float_value": 0.4} + return [{"float_value": 0.4}] * len(retrieved_batches[0]) + + +class CompoundStrictBatchBlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "short_description": "", + "long_description": "", + "license": "Apache-2.0", + "block_type": "dummy", + } + ) + type: Literal["CompoundStrictBatchBlock"] + compound_parameter: Dict[str, Selector()] + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["compound_parameter"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="float_value", + kind=[FLOAT_ZERO_TO_ONE_KIND], + ), + ] + + +class CompoundStrictBatchBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return CompoundStrictBatchBlockManifest + + def run(self, compound_parameter: Dict[str, Any]) -> BlockResult: + retrieved_batches = [ + v for v in compound_parameter.values() if isinstance(v, Batch) + ] + return [{"float_value": 0.4}] * len(retrieved_batches[0]) + + +class CompoundNonStrictBatchBlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "short_description": "", + "long_description": "", + "license": "Apache-2.0", + "block_type": "dummy", + } + ) + type: Literal["CompoundNonStrictBatchBlock"] + compound_parameter: Dict[str, Union[Selector()]] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="float_value", + kind=[FLOAT_ZERO_TO_ONE_KIND], + ), + ] + + +class CompoundNonStrictBatchBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return CompoundNonStrictBatchBlockManifest + + def run(self, compound_parameter: Dict[str, Any]) -> BlockResult: + return {"float_value": 0.4} + + +def load_blocks() -> List[Type[WorkflowBlock]]: + return [ + NonBatchInputBlock, + MixedInputWithBatchesBlock, + MixedInputWithoutBatchesBlock, + BatchInputProcessingBatchesBlock, + BatchInputNotProcessingBatchesBlock, + CompoundNonBatchInputBlock, + CompoundMixedInputBlock, + CompoundStrictBatchBlock, + CompoundNonStrictBatchBlock, + ] diff --git a/tests/workflows/integration_tests/execution/stub_plugins/scalar_selectors_plugin/__init__.py b/tests/workflows/integration_tests/execution/stub_plugins/scalar_selectors_plugin/__init__.py new file mode 100644 index 000000000..8a708bb3e --- /dev/null +++ b/tests/workflows/integration_tests/execution/stub_plugins/scalar_selectors_plugin/__init__.py @@ -0,0 +1,175 @@ +from typing import Any, List, Literal, Optional, Type, Union +from uuid import uuid4 + +import numpy as np +from pydantic import Field + +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + IMAGE_KIND, + LIST_OF_VALUES_KIND, + STRING_KIND, + Selector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) + + +class SecretBlockManifest(WorkflowBlockManifest): + type: Literal["secret_store"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="secret", kind=[STRING_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class SecretStoreBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return SecretBlockManifest + + def run(self) -> BlockResult: + return {"secret": "my_secret"} + + +class BlockManifest(WorkflowBlockManifest): + type: Literal["secret_store_user"] + image: Selector(kind=[IMAGE_KIND]) = Field( + title="Input Image", + description="The input image for this step.", + ) + secret: Selector(kind=[STRING_KIND]) + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["image"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="output", kind=[STRING_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class SecretStoreUserBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run(self, image: Batch[WorkflowImageData], secret: str) -> BlockResult: + return [{"output": secret}] * len(image) + + +class BatchSecretBlockManifest(WorkflowBlockManifest): + type: Literal["batch_secret_store"] + image: Selector(kind=[IMAGE_KIND]) = Field( + title="Input Image", + description="The input image for this step.", + ) + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="secret", kind=[STRING_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class BatchSecretStoreBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BatchSecretBlockManifest + + def run(self, image: WorkflowImageData) -> BlockResult: + return {"secret": f"my_secret_{uuid4()}"} + + +class NonBatchSecretStoreUserBlockManifest(WorkflowBlockManifest): + type: Literal["non_batch_secret_store_user"] + secret: Selector(kind=[STRING_KIND]) + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="output", kind=[STRING_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class NonBatchSecretStoreUserBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return NonBatchSecretStoreUserBlockManifest + + def run(self, secret: str) -> BlockResult: + return {"output": secret} + + +class BlockWithReferenceImagesManifest(WorkflowBlockManifest): + type: Literal["reference_images_comparison"] + image: Selector(kind=[IMAGE_KIND]) + reference_images: Union[Selector(kind=[LIST_OF_VALUES_KIND]), Any] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="similarity", kind=[LIST_OF_VALUES_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +class BlockWithReferenceImagesBlock(WorkflowBlock): + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockWithReferenceImagesManifest + + def run( + self, image: WorkflowImageData, reference_images: List[np.ndarray] + ) -> BlockResult: + similarity = [] + for ref_image in reference_images: + similarity.append( + (image.numpy_image == ref_image).sum() / image.numpy_image.size + ) + return {"similarity": similarity} + + +def load_blocks() -> List[Type[WorkflowBlock]]: + return [ + SecretStoreBlock, + SecretStoreUserBlock, + BatchSecretStoreBlock, + NonBatchSecretStoreUserBlock, + BlockWithReferenceImagesBlock, + ] diff --git a/tests/workflows/integration_tests/execution/test_workflow_detection_plus_classification.py b/tests/workflows/integration_tests/execution/test_workflow_detection_plus_classification.py index b72d7ed64..10b610b45 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_detection_plus_classification.py +++ b/tests/workflows/integration_tests/execution/test_workflow_detection_plus_classification.py @@ -9,7 +9,7 @@ add_to_workflows_gallery, ) -DETECTION_PLUS_CLASSIFICATION_WORKFLOW = { +LEGACY_DETECTION_PLUS_CLASSIFICATION_WORKFLOW = { "version": "1.0", "inputs": [{"type": "WorkflowImage", "name": "image"}], "steps": [ @@ -43,6 +43,78 @@ } +def test_legacy_detection_plus_classification_workflow_when_minimal_valid_input_provided( + model_manager: ModelManager, + dogs_image: np.ndarray, + roboflow_api_key: str, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": roboflow_api_key, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=LEGACY_DETECTION_PLUS_CLASSIFICATION_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": dogs_image, + } + ) + + assert isinstance(result, list), "Expected list to be delivered" + assert len(result) == 1, "Expected 1 element in the output for one input image" + assert set(result[0].keys()) == { + "predictions", + }, "Expected all declared outputs to be delivered" + assert ( + len(result[0]["predictions"]) == 2 + ), "Expected 2 dogs crops on input image, hence 2 nested classification results" + assert [result[0]["predictions"][0]["top"], result[0]["predictions"][1]["top"]] == [ + "116.Parson_russell_terrier", + "131.Wirehaired_pointing_griffon", + ], "Expected predictions to be as measured in reference run" + + +DETECTION_PLUS_CLASSIFICATION_WORKFLOW_V2_BLOCKS = { + "version": "1.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}], + "steps": [ + { + "type": "roboflow_core/roboflow_object_detection_model@v2", + "name": "general_detection", + "image": "$inputs.image", + "model_id": "yolov8n-640", + "class_filter": ["dog"], + }, + { + "type": "roboflow_core/dynamic_crop@v1", + "name": "cropping", + "image": "$inputs.image", + "predictions": "$steps.general_detection.predictions", + }, + { + "type": "roboflow_core/roboflow_classification_model@v2", + "name": "breds_classification", + "image": "$steps.cropping.crops", + "model_id": "dog-breed-xpaq6/1", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "predictions", + "selector": "$steps.breds_classification.predictions", + }, + ], +} + + @add_to_workflows_gallery( category="Workflows with multiple models", use_case_title="Workflow detection model followed by classifier", @@ -60,7 +132,7 @@ Secondary model is supposed to make prediction from dogs breed classifier model to assign detailed class for each dog instance. """, - workflow_definition=DETECTION_PLUS_CLASSIFICATION_WORKFLOW, + workflow_definition=DETECTION_PLUS_CLASSIFICATION_WORKFLOW_V2_BLOCKS, workflow_name_in_app="detection-plus-classification", ) def test_detection_plus_classification_workflow_when_minimal_valid_input_provided( @@ -75,7 +147,7 @@ def test_detection_plus_classification_workflow_when_minimal_valid_input_provide "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, } execution_engine = ExecutionEngine.init( - workflow_definition=DETECTION_PLUS_CLASSIFICATION_WORKFLOW, + workflow_definition=DETECTION_PLUS_CLASSIFICATION_WORKFLOW_V2_BLOCKS, init_parameters=workflow_init_parameters, max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, ) @@ -113,7 +185,7 @@ def test_detection_plus_classification_workflow_when_nothing_gets_predicted( "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, } execution_engine = ExecutionEngine.init( - workflow_definition=DETECTION_PLUS_CLASSIFICATION_WORKFLOW, + workflow_definition=DETECTION_PLUS_CLASSIFICATION_WORKFLOW_V2_BLOCKS, init_parameters=workflow_init_parameters, max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, ) @@ -140,14 +212,14 @@ def test_detection_plus_classification_workflow_when_nothing_gets_predicted( "inputs": [{"type": "WorkflowImage", "name": "image"}], "steps": [ { - "type": "ObjectDetectionModel", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "general_detection", "image": "$inputs.image", "model_id": "yolov8n-640", "class_filter": ["dog"], }, { - "type": "DetectionsConsensus", + "type": "roboflow_core/detections_consensus@v1", "name": "detections_consensus", "predictions_batches": [ "$steps.general_detection.predictions", @@ -155,13 +227,13 @@ def test_detection_plus_classification_workflow_when_nothing_gets_predicted( "required_votes": 1, }, { - "type": "Crop", + "type": "roboflow_core/dynamic_crop@v1", "name": "cropping", "image": "$inputs.image", "predictions": "$steps.detections_consensus.predictions", }, { - "type": "ClassificationModel", + "type": "roboflow_core/roboflow_classification_model@v2", "name": "breds_classification", "image": "$steps.cropping.crops", "model_id": "dog-breed-xpaq6/1", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_active_learning_sink.py b/tests/workflows/integration_tests/execution/test_workflow_with_active_learning_sink.py index f7b3df4ab..cbee95f50 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_active_learning_sink.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_active_learning_sink.py @@ -30,7 +30,7 @@ ], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "general_detection", "image": "$inputs.image", "model_id": "yolov8n-640", @@ -43,7 +43,7 @@ "predictions": "$steps.general_detection.predictions", }, { - "type": "roboflow_core/roboflow_classification_model@v1", + "type": "roboflow_core/roboflow_classification_model@v2", "name": "breds_classification", "image": "$steps.cropping.crops", "model_id": "dog-breed-xpaq6/1", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_arbitrary_batch_inputs.py b/tests/workflows/integration_tests/execution/test_workflow_with_arbitrary_batch_inputs.py new file mode 100644 index 000000000..51802efef --- /dev/null +++ b/tests/workflows/integration_tests/execution/test_workflow_with_arbitrary_batch_inputs.py @@ -0,0 +1,1990 @@ +from unittest import mock +from unittest.mock import MagicMock + +import numpy as np +import pytest +import supervision as sv + +from inference.core.env import WORKFLOWS_MAX_CONCURRENT_STEPS +from inference.core.managers.base import ModelManager +from inference.core.utils.image_utils import load_image +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.errors import ( + AssumptionError, + ExecutionGraphStructureError, + RuntimeInputError, +) +from inference.core.workflows.execution_engine.core import ExecutionEngine +from inference.core.workflows.execution_engine.introspection import blocks_loader + +TWO_STAGE_WORKFLOW = { + "version": "1.3.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}], + "steps": [ + { + "type": "ObjectDetectionModel", + "name": "general_detection", + "image": "$inputs.image", + "model_id": "yolov8n-640", + "class_filter": ["dog"], + }, + { + "type": "Crop", + "name": "cropping", + "image": "$inputs.image", + "predictions": "$steps.general_detection.predictions", + }, + { + "type": "ClassificationModel", + "name": "breds_classification", + "image": "$steps.cropping.crops", + "model_id": "dog-breed-xpaq6/1", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "predictions", + "selector": "$steps.breds_classification.predictions", + }, + ], +} + + +OBJECT_DETECTION_WORKFLOW = { + "version": "1.3.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}], + "steps": [ + { + "type": "ObjectDetectionModel", + "name": "general_detection", + "image": "$inputs.image", + "model_id": "yolov8n-640", + "class_filter": ["dog"], + } + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.general_detection.*", + }, + ], +} + + +CROP_WORKFLOW = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + { + "type": "WorkflowBatchInput", + "name": "predictions", + "kind": ["object_detection_prediction"], + }, + ], + "steps": [ + { + "type": "Crop", + "name": "cropping", + "image": "$inputs.image", + "predictions": "$inputs.predictions", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.cropping.*", + }, + ], +} + +CLASSIFICATION_WORKFLOW = { + "version": "1.3.0", + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "crops", + "kind": ["image"], + "dimensionality": 2, + }, + ], + "steps": [ + { + "type": "ClassificationModel", + "name": "breds_classification", + "image": "$inputs.crops", + "model_id": "dog-breed-xpaq6/1", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "predictions", + "selector": "$steps.breds_classification.predictions", + }, + ], +} + + +def test_debug_execution_of_workflow_for_single_image_without_conditional_evaluation( + model_manager: ModelManager, + dogs_image: np.ndarray, + roboflow_api_key: str, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": roboflow_api_key, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + end_to_end_execution_engine = ExecutionEngine.init( + workflow_definition=TWO_STAGE_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + first_step_execution_engine = ExecutionEngine.init( + workflow_definition=OBJECT_DETECTION_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + second_step_execution_engine = ExecutionEngine.init( + workflow_definition=CROP_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + third_step_execution_engine = ExecutionEngine.init( + workflow_definition=CLASSIFICATION_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + e2e_results = end_to_end_execution_engine.run( + runtime_parameters={ + "image": dogs_image, + } + ) + detection_results = first_step_execution_engine.run( + runtime_parameters={ + "image": dogs_image, + } + ) + cropping_results = second_step_execution_engine.run( + runtime_parameters={ + "image": dogs_image, + "predictions": detection_results[0]["result"]["predictions"], + } + ) + classification_results = third_step_execution_engine.run( + runtime_parameters={ + "crops": [[e["crops"] for e in cropping_results[0]["result"]]], + } + ) + + # then + e2e_top_classes = [p["top"] for p in e2e_results[0]["predictions"]] + debug_top_classes = [p["top"] for p in classification_results[0]["predictions"]] + assert ( + e2e_top_classes == debug_top_classes + ), "Expected top class prediction from step-by-step execution to match e2e execution" + e2e_confidence = [p["confidence"] for p in e2e_results[0]["predictions"]] + debug_confidence = [ + p["confidence"] for p in classification_results[0]["predictions"] + ] + assert np.allclose( + e2e_confidence, debug_confidence, atol=1e-4 + ), "Expected confidences from step-by-step execution to match e2e execution" + + +def test_debug_execution_of_workflow_for_single_image_without_conditional_evaluation_when_serialization_is_requested( + model_manager: ModelManager, + dogs_image: np.ndarray, + roboflow_api_key: str, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": roboflow_api_key, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + end_to_end_execution_engine = ExecutionEngine.init( + workflow_definition=TWO_STAGE_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + first_step_execution_engine = ExecutionEngine.init( + workflow_definition=OBJECT_DETECTION_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + second_step_execution_engine = ExecutionEngine.init( + workflow_definition=CROP_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + third_step_execution_engine = ExecutionEngine.init( + workflow_definition=CLASSIFICATION_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + e2e_results = end_to_end_execution_engine.run( + runtime_parameters={ + "image": dogs_image, + }, + serialize_results=True, + ) + detection_results = first_step_execution_engine.run( + runtime_parameters={ + "image": dogs_image, + }, + serialize_results=True, + ) + detection_results_not_serialized = first_step_execution_engine.run( + runtime_parameters={ + "image": dogs_image, + }, + ) + cropping_results = second_step_execution_engine.run( + runtime_parameters={ + "image": dogs_image, + "predictions": detection_results[0]["result"]["predictions"], + }, + serialize_results=True, + ) + cropping_results_not_serialized = second_step_execution_engine.run( + runtime_parameters={ + "image": dogs_image, + "predictions": detection_results_not_serialized[0]["result"]["predictions"], + }, + serialize_results=False, + ) + classification_results = third_step_execution_engine.run( + runtime_parameters={ + "crops": [[e["crops"] for e in cropping_results[0]["result"]]], + }, + serialize_results=True, + ) + + # then + assert isinstance( + detection_results[0]["result"]["predictions"], dict + ), "Expected sv.Detections to be serialized" + assert isinstance( + detection_results_not_serialized[0]["result"]["predictions"], sv.Detections + ), "Expected sv.Detections not to be serialized" + deserialized_detections = sv.Detections.from_inference( + detection_results[0]["result"]["predictions"] + ) + assert np.allclose( + deserialized_detections.confidence, + detection_results_not_serialized[0]["result"]["predictions"].confidence, + atol=1e-4, + ), "Expected confidence match when serialized detections are deserialized" + intermediate_crop = cropping_results[0]["result"][0]["crops"] + assert ( + intermediate_crop["type"] == "base64" + ), "Expected crop to be serialized to base64" + decoded_image, _ = load_image(intermediate_crop) + number_of_pixels = ( + decoded_image.shape[0] * decoded_image.shape[1] * decoded_image.shape[2] + ) + assert ( + decoded_image.shape + == cropping_results_not_serialized[0]["result"][0]["crops"].numpy_image.shape + ), "Expected deserialized crop to match in size with not serialized one" + assert ( + abs( + (decoded_image.sum() / number_of_pixels) + - ( + cropping_results_not_serialized[0]["result"][0][ + "crops" + ].numpy_image.sum() + / number_of_pixels + ) + ) + < 1e-1 + ), "Content of serialized and not serialized crop should roughly match (up to compression)" + e2e_top_classes = [p["top"] for p in e2e_results[0]["predictions"]] + debug_top_classes = [p["top"] for p in classification_results[0]["predictions"]] + assert ( + e2e_top_classes == debug_top_classes + ), "Expected top class prediction from step-by-step execution to match e2e execution" + e2e_confidence = [p["confidence"] for p in e2e_results[0]["predictions"]] + debug_confidence = [ + p["confidence"] for p in classification_results[0]["predictions"] + ] + assert np.allclose( + e2e_confidence, debug_confidence, atol=1e-1 + ), "Expected confidences from step-by-step execution to match e2e execution" + + +def test_debug_execution_of_workflow_for_batch_of_images_without_conditional_evaluation( + model_manager: ModelManager, + dogs_image: np.ndarray, + roboflow_api_key: str, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": roboflow_api_key, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + end_to_end_execution_engine = ExecutionEngine.init( + workflow_definition=TWO_STAGE_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + first_step_execution_engine = ExecutionEngine.init( + workflow_definition=OBJECT_DETECTION_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + second_step_execution_engine = ExecutionEngine.init( + workflow_definition=CROP_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + third_step_execution_engine = ExecutionEngine.init( + workflow_definition=CLASSIFICATION_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + e2e_results = end_to_end_execution_engine.run( + runtime_parameters={ + "image": [dogs_image, dogs_image], + } + ) + detection_results = first_step_execution_engine.run( + runtime_parameters={ + "image": [dogs_image, dogs_image], + } + ) + cropping_results = second_step_execution_engine.run( + runtime_parameters={ + "image": [dogs_image, dogs_image], + "predictions": [ + detection_results[0]["result"]["predictions"], + detection_results[1]["result"]["predictions"], + ], + } + ) + classification_results = third_step_execution_engine.run( + runtime_parameters={ + "crops": [ + [e["crops"] for e in cropping_results[0]["result"]], + [e["crops"] for e in cropping_results[1]["result"]], + ], + } + ) + + # then + e2e_top_classes = [p["top"] for r in e2e_results for p in r["predictions"]] + debug_top_classes = [ + p["top"] for r in classification_results for p in r["predictions"] + ] + assert ( + e2e_top_classes == debug_top_classes + ), "Expected top class prediction from step-by-step execution to match e2e execution" + e2e_confidence = [p["confidence"] for r in e2e_results for p in r["predictions"]] + debug_confidence = [ + p["confidence"] for r in classification_results for p in r["predictions"] + ] + assert np.allclose( + e2e_confidence, debug_confidence, atol=1e-4 + ), "Expected confidences from step-by-step execution to match e2e execution" + + +TWO_STAGE_WORKFLOW_WITH_FLOW_CONTROL = { + "version": "1.3.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}], + "steps": [ + { + "type": "ObjectDetectionModel", + "name": "general_detection", + "image": "$inputs.image", + "model_id": "yolov8n-640", + "class_filter": ["dog"], + }, + { + "type": "Crop", + "name": "cropping", + "image": "$inputs.image", + "predictions": "$steps.general_detection.predictions", + }, + { + "type": "roboflow_core/continue_if@v1", + "name": "verify_crop_size", + "condition_statement": { + "type": "StatementGroup", + "statements": [ + { + "type": "BinaryStatement", + "left_operand": { + "type": "DynamicOperand", + "operand_name": "crops", + "operations": [ + { + "type": "ExtractImageProperty", + "property_name": "size", + }, + ], + }, + "comparator": {"type": "(Number) >="}, + "right_operand": { + "type": "StaticOperand", + "value": 48000, + }, + } + ], + }, + "next_steps": ["$steps.breds_classification"], + "evaluation_parameters": {"crops": "$steps.cropping.crops"}, + }, + { + "type": "ClassificationModel", + "name": "breds_classification", + "image": "$steps.cropping.crops", + "model_id": "dog-breed-xpaq6/1", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "predictions", + "selector": "$steps.breds_classification.predictions", + }, + ], +} + + +def test_debug_execution_of_workflow_for_batch_of_images_with_conditional_evaluation( + model_manager: ModelManager, + dogs_image: np.ndarray, + roboflow_api_key: str, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": roboflow_api_key, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + end_to_end_execution_engine = ExecutionEngine.init( + workflow_definition=TWO_STAGE_WORKFLOW_WITH_FLOW_CONTROL, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + first_step_execution_engine = ExecutionEngine.init( + workflow_definition=OBJECT_DETECTION_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + second_step_execution_engine = ExecutionEngine.init( + workflow_definition=CROP_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + third_step_execution_engine = ExecutionEngine.init( + workflow_definition=CLASSIFICATION_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + e2e_results = end_to_end_execution_engine.run( + runtime_parameters={ + "image": [dogs_image, dogs_image], + } + ) + detection_results = first_step_execution_engine.run( + runtime_parameters={ + "image": [dogs_image, dogs_image], + } + ) + cropping_results = second_step_execution_engine.run( + runtime_parameters={ + "image": [dogs_image, dogs_image], + "predictions": [ + detection_results[0]["result"]["predictions"], + detection_results[1]["result"]["predictions"], + ], + } + ) + classification_results = third_step_execution_engine.run( + runtime_parameters={ + "crops": [ + [cropping_results[0]["result"][0]["crops"], None], + [cropping_results[1]["result"][0]["crops"], None], + ], + } + ) + + # then + assert ( + e2e_results[0]["predictions"][0] is not None + ), "Expected first dog crop not to be excluded by conditional eval" + assert ( + e2e_results[0]["predictions"][1] is None + ), "Expected second dog crop to be excluded by conditional eval" + assert ( + e2e_results[1]["predictions"][0] is not None + ), "Expected first dog crop not to be excluded by conditional eval" + assert ( + e2e_results[1]["predictions"][1] is None + ), "Expected second dog crop to be excluded by conditional eval" + e2e_top_classes = [ + p["top"] if p else None for r in e2e_results for p in r["predictions"] + ] + debug_top_classes = [ + p["top"] if p else None + for r in classification_results + for p in r["predictions"] + ] + assert ( + e2e_top_classes == debug_top_classes + ), "Expected top class prediction from step-by-step execution to match e2e execution" + e2e_confidence = [ + p["confidence"] if p else -1000.0 for r in e2e_results for p in r["predictions"] + ] + debug_confidence = [ + p["confidence"] if p else -1000.0 + for r in classification_results + for p in r["predictions"] + ] + assert np.allclose( + e2e_confidence, debug_confidence, atol=1e-4 + ), "Expected confidences from step-by-step execution to match e2e execution" + + +def test_debug_execution_when_empty_batch_oriented_input_provided( + model_manager: ModelManager, + dogs_image: np.ndarray, + roboflow_api_key: str, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": roboflow_api_key, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=CROP_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + with pytest.raises(RuntimeInputError): + _ = execution_engine.run( + runtime_parameters={"image": [dogs_image, dogs_image], "predictions": None} + ) + + +WORKFLOW_WITH_BATCH_ORIENTED_CONFIDENCE = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + { + "type": "WorkflowBatchInput", + "name": "confidence", + }, + ], + "steps": [ + { + "type": "ObjectDetectionModel", + "name": "general_detection", + "image": "$inputs.image", + "model_id": "yolov8n-640", + "class_filter": ["dog"], + "confidence": "$inputs.confidence", + } + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.general_detection.*", + }, + ], +} + + +def test_workflow_run_which_hooks_up_batch_oriented_input_into_non_batch_oriented_parameters( + model_manager: ModelManager, + dogs_image: np.ndarray, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + + # when + with pytest.raises(ExecutionGraphStructureError): + _ = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_CONFIDENCE, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_NON_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "MixedInputWithBatchesBlock", + "name": "step_two", + "mixed_parameter": "$steps.step_one.float_value", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_two.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_step_feeds_non_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_NON_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "non_batch_parameter": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_BATCH_ORIENTED_STEP_NOT_OPERATING_BATCH_WISE = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "BatchInputBlockNotProcessingBatches", + "name": "step_two", + "batch_parameter": "$steps.step_one.float_value", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_two.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_step_feeds_batch_oriented_step_not_operating_batch_wise( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_BATCH_ORIENTED_STEP_NOT_OPERATING_BATCH_WISE, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "non_batch_parameter": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_BATCH_ORIENTED_STEP_OPERATING_BATCH_WISE = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "BatchInputBlockProcessingBatches", + "name": "step_two", + "batch_parameter": "$steps.step_one.float_value", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_two.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_step_feeds_batch_oriented_step_operating_batch_wise( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + + # when + with pytest.raises(ExecutionGraphStructureError): + _ = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_BATCH_ORIENTED_STEP_OPERATING_BATCH_WISE, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_MIXED_INPUT_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "MixedInputWithBatchesBlock", + "name": "step_two", + "mixed_parameter": "$steps.step_one.float_value", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_two.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_step_feeds_mixed_input_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_MIXED_INPUT_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "non_batch_parameter": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_BATCH_ORIENTED_STEP_FEEDING_MIXED_INPUT_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "BatchInputBlockNotProcessingBatches", + "name": "step_two", + "batch_parameter": "$steps.step_one.float_value", + }, + { + "type": "MixedInputWithBatchesBlock", + "name": "step_three", + "mixed_parameter": "$steps.step_two.float_value", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_three.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_step_feeds_mixed_input_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_STEP_FEEDING_MIXED_INPUT_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "non_batch_parameter": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_INTO_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "data", + }, + ], + "steps": [ + { + "type": "BatchInputBlockProcessingBatches", + "name": "step_one", + "batch_parameter": "$inputs.data", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_input_feeds_batch_input_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_INTO_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "data": ["some", "other"], + } + ) + + # then + assert len(result) == 2, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + assert result[1]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_INTO_MIXED_INPUT_STEP = { + "version": "1.3.0", + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "data", + }, + ], + "steps": [ + { + "type": "MixedInputWithBatchesBlock", + "name": "step_one", + "mixed_parameter": "$inputs.data", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_input_feeds_mixed_input_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_INTO_MIXED_INPUT_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "data": ["some", "other"], + } + ) + + # then + assert len(result) == 2, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + assert result[1]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_INTO_NON_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "data", + }, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.data", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_input_feeds_non_batch_input_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_INTO_NON_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "data": ["some", "other"], + } + ) + + # then + assert len(result) == 2, "Expected two outputs for two input elements" + assert result[0]["result"] == 0.4, "Expected hardcoded value" + assert result[1]["result"] == 0.4, "Expected hardcoded value" + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_COMPOUND_NON_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "CompoundNonBatchInputBlock", + "name": "step_two", + "compound_parameter": { + "some": "$steps.step_one.float_value", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_two.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_step_feeds_compound_non_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_COMPOUND_NON_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "non_batch_parameter": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_COMPOUND_MIXED_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "CompoundMixedInputBlockManifestBlock", + "name": "step_two", + "compound_parameter": { + "some": "$steps.step_one.float_value", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_two.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_step_feeds_compound_mixed_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_COMPOUND_MIXED_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "non_batch_parameter": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_COMPOUND_LOOSELY_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "CompoundNonStrictBatchBlock", + "name": "step_two", + "compound_parameter": { + "some": "$steps.step_one.float_value", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_two.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_step_feeds_compound_loosely_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_COMPOUND_LOOSELY_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "non_batch_parameter": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_COMPOUND_STRICTLY_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "CompoundStrictBatchBlock", + "name": "step_two", + "compound_parameter": { + "some": "$steps.step_one.float_value", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_two.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_step_feeds_compound_strictly_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + + # then + with pytest.raises(ExecutionGraphStructureError): + _ = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_STEP_FEEDING_COMPOUND_STRICTLY_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + +WORKFLOW_WITH_BATCH_ORIENTED_STEP_FEEDING_COMPOUND_NON_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "BatchInputBlockNotProcessingBatches", + "name": "step_two", + "batch_parameter": "$steps.step_one.float_value", + }, + { + "type": "CompoundNonBatchInputBlock", + "name": "step_three", + "compound_parameter": { + "some": "$steps.step_two.float_value", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_three.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_step_feeds_compound_non_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_STEP_FEEDING_COMPOUND_NON_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "non_batch_parameter": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_BATCH_ORIENTED_STEP_FEEDING_MIXED_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "BatchInputBlockNotProcessingBatches", + "name": "step_two", + "batch_parameter": "$steps.step_one.float_value", + }, + { + "type": "CompoundMixedInputBlockManifestBlock", + "name": "step_three", + "compound_parameter": { + "some": "$steps.step_two.float_value", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_three.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_step_feeds_compound_mixed_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_STEP_FEEDING_MIXED_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "non_batch_parameter": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_BATCH_ORIENTED_STEP_FEEDING_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "non_batch_parameter"}, + ], + "steps": [ + { + "type": "NonBatchInputBlock", + "name": "step_one", + "non_batch_parameter": "$inputs.non_batch_parameter", + }, + { + "type": "BatchInputBlockNotProcessingBatches", + "name": "step_two", + "batch_parameter": "$steps.step_one.float_value", + }, + { + "type": "CompoundNonStrictBatchBlock", + "name": "step_three", + "compound_parameter": { + "some": "$steps.step_two.float_value", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_three.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_step_feeds_compound_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_STEP_FEEDING_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "non_batch_parameter": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_NON_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "data"}, + ], + "steps": [ + { + "type": "CompoundNonBatchInputBlock", + "name": "step_one", + "compound_parameter": { + "some": "$inputs.data", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_input_feeds_compound_non_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_NON_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "data": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_MIXED_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "data"}, + ], + "steps": [ + { + "type": "CompoundMixedInputBlockManifestBlock", + "name": "step_one", + "compound_parameter": { + "some": "$inputs.data", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_input_feeds_compound_mixed_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_MIXED_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "data": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_LOOSELY_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "data"}, + ], + "steps": [ + { + "type": "CompoundNonStrictBatchBlock", + "name": "step_one", + "compound_parameter": { + "some": "$inputs.data", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_input_feeds_compound_loosely_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_LOOSELY_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # then + result = execution_engine.run( + runtime_parameters={ + "data": "some", + } + ) + + # then + assert len(result) == 1, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_NON_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_STRICTLY_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowParameter", "name": "data"}, + ], + "steps": [ + { + "type": "CompoundStrictBatchBlock", + "name": "step_one", + "compound_parameter": { + "some": "$inputs.data", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_non_batch_oriented_input_feeds_compound_strictly_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + + # then + with pytest.raises(ExecutionGraphStructureError): + _ = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_NON_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_STRICTLY_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + +WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_NON_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "data", + }, + ], + "steps": [ + { + "type": "CompoundNonBatchInputBlock", + "name": "step_one", + "compound_parameter": { + "some": "$inputs.data", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_input_feeds_compound_non_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_NON_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "data": ["some", "other"], + } + ) + + # then + assert len(result) == 2, "Expected 2 outputs for 2 inputs" + assert result[0]["result"] == 0.4, "Expected hardcoded value" + assert result[1]["result"] == 0.4, "Expected hardcoded value" + + +WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_MIXED_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "data", + }, + ], + "steps": [ + { + "type": "CompoundMixedInputBlockManifestBlock", + "name": "step_one", + "compound_parameter": { + "some": "$inputs.data", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_input_feeds_compound_mixed_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_MIXED_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "data": ["some", "other"], + } + ) + + # then + assert len(result) == 2, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + assert result[1]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_LOOSELY_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "data", + }, + ], + "steps": [ + { + "type": "CompoundNonStrictBatchBlock", + "name": "step_one", + "compound_parameter": { + "some": "$inputs.data", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_input_feeds_compound_loosely_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_LOOSELY_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "data": ["some", "other"], + } + ) + + # then + assert len(result) == 2, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + assert result[1]["result"] == 0.4, "Expected hardcoded result" + + +WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_STRICTLY_BATCH_ORIENTED_STEP = { + "version": "1.3.0", + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "data", + }, + ], + "steps": [ + { + "type": "CompoundStrictBatchBlock", + "name": "step_one", + "compound_parameter": { + "some": "$inputs.data", + }, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.step_one.float_value", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_when_batch_oriented_input_feeds_compound_strictly_batch_oriented_step( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.mixed_input_characteristic_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_BATCH_ORIENTED_INPUT_FEEDING_COMPOUND_STRICTLY_BATCH_ORIENTED_STEP, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "data": ["some", "other"], + } + ) + + # then + assert len(result) == 2, "Expected singular result" + assert result[0]["result"] == 0.4, "Expected hardcoded result" + assert result[1]["result"] == 0.4, "Expected hardcoded result" diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_claude_models.py b/tests/workflows/integration_tests/execution/test_workflow_with_claude_models.py index d55516aa5..ddac3b424 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_claude_models.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_claude_models.py @@ -302,6 +302,100 @@ def test_workflow_with_captioning_prompt( ), "Expected non-empty string generated" +CLASSIFICATION_WORKFLOW_WITH_LEGACY_PARSER = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + {"type": "WorkflowParameter", "name": "api_key"}, + {"type": "WorkflowParameter", "name": "classes"}, + ], + "steps": [ + { + "type": "roboflow_core/anthropic_claude@v1", + "name": "claude", + "images": "$inputs.image", + "task_type": "classification", + "classes": "$inputs.classes", + "api_key": "$inputs.api_key", + }, + { + "type": "roboflow_core/vlm_as_classifier@v2", + "name": "parser", + "image": "$inputs.image", + "vlm_output": "$steps.claude.output", + "classes": "$steps.claude.classes", + }, + { + "type": "roboflow_core/property_definition@v1", + "name": "top_class", + "operations": [ + {"type": "ClassificationPropertyExtract", "property_name": "top_class"} + ], + "data": "$steps.parser.predictions", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "claude_result", + "selector": "$steps.claude.output", + }, + { + "type": "JsonField", + "name": "top_class", + "selector": "$steps.top_class.output", + }, + { + "type": "JsonField", + "name": "parsed_prediction", + "selector": "$steps.parser.*", + }, + ], +} + + +@pytest.mark.skipif( + condition=ANTHROPIC_API_KEY is None, reason="Anthropic API key not provided" +) +def test_workflow_with_multi_class_classifier_prompt_with_legacy_parser( + model_manager: ModelManager, + dogs_image: np.ndarray, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=CLASSIFICATION_WORKFLOW_WITH_LEGACY_PARSER, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [dogs_image], + "api_key": ANTHROPIC_API_KEY, + "classes": ["cat", "dog"], + } + ) + + # then + assert len(result) == 1, "Single image given, expected single output" + assert set(result[0].keys()) == { + "claude_result", + "top_class", + "parsed_prediction", + }, "Expected all outputs to be delivered" + assert ( + isinstance(result[0]["claude_result"], str) + and len(result[0]["claude_result"]) > 0 + ), "Expected non-empty string generated" + assert result[0]["top_class"] == "dog" + assert result[0]["parsed_prediction"]["error_status"] is False + + CLASSIFICATION_WORKFLOW = { "version": "1.0", "inputs": [ @@ -319,7 +413,7 @@ def test_workflow_with_captioning_prompt( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$inputs.image", "vlm_output": "$steps.claude.output", @@ -359,7 +453,7 @@ def test_workflow_with_captioning_prompt( use_case_title="Using Anthropic Claude as multi-class classifier", use_case_description=""" In this example, Anthropic Claude model is used as classifier. Output from the model is parsed by -special `roboflow_core/vlm_as_classifier@v1` block which turns model output text into +special `roboflow_core/vlm_as_classifier@v2` block which turns model output text into full-blown prediction, which can later be used by other blocks compatible with classification predictions - in this case we extract top-class property. """, @@ -425,7 +519,7 @@ def test_workflow_with_multi_class_classifier_prompt( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$inputs.image", # requires image input to construct valid output compatible with "inference" "vlm_output": "$steps.claude.output", @@ -460,7 +554,7 @@ def test_workflow_with_multi_class_classifier_prompt( use_case_title="Using Anthropic Claude as multi-label classifier", use_case_description=""" In this example, Anthropic Claude model is used as multi-label classifier. Output from the model is parsed by -special `roboflow_core/vlm_as_classifier@v1` block which turns model output text into +special `roboflow_core/vlm_as_classifier@v2` block which turns model output text into full-blown prediction, which can later be used by other blocks compatible with classification predictions - in this case we extract top-class property. """, @@ -589,7 +683,7 @@ def test_workflow_with_structured_prompt( assert result[0]["result"] == "2" -OBJECT_DETECTION_WORKFLOW = { +OBJECT_DETECTION_WORKFLOW_LEGACY_PARSER = { "version": "1.0", "inputs": [ {"type": "WorkflowImage", "name": "image"}, @@ -630,6 +724,86 @@ def test_workflow_with_structured_prompt( } +@pytest.mark.skipif( + condition=ANTHROPIC_API_KEY is None, reason="Anthropic API key not provided" +) +def test_workflow_with_object_detection_prompt_when_legacy_parser_in_use( + model_manager: ModelManager, + dogs_image: np.ndarray, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=OBJECT_DETECTION_WORKFLOW_LEGACY_PARSER, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [dogs_image], + "api_key": ANTHROPIC_API_KEY, + "classes": ["cat", "dog"], + } + ) + + # then + assert len(result) == 1, "Single image given, expected single output" + assert set(result[0].keys()) == { + "claude_result", + "parsed_prediction", + }, "Expected all outputs to be delivered" + assert result[0]["parsed_prediction"].data["class_name"].tolist() == [ + "dog", + "dog", + ], "Expected 2 dogs to be detected" + + +OBJECT_DETECTION_WORKFLOW = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + {"type": "WorkflowParameter", "name": "api_key"}, + {"type": "WorkflowParameter", "name": "classes"}, + ], + "steps": [ + { + "type": "roboflow_core/anthropic_claude@v1", + "name": "claude", + "images": "$inputs.image", + "task_type": "object-detection", + "classes": "$inputs.classes", + "api_key": "$inputs.api_key", + }, + { + "type": "roboflow_core/vlm_as_detector@v2", + "name": "parser", + "vlm_output": "$steps.claude.output", + "image": "$inputs.image", + "classes": "$steps.claude.classes", + "model_type": "anthropic-claude", + "task_type": "object-detection", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "claude_result", + "selector": "$steps.claude.output", + }, + { + "type": "JsonField", + "name": "parsed_prediction", + "selector": "$steps.parser.predictions", + }, + ], +} + + @add_to_workflows_gallery( category="Workflows with Visual Language Models", use_case_title="Using Anthropic Claude as object-detection model", @@ -718,7 +892,7 @@ def test_workflow_with_object_detection_prompt( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$steps.cropping.crops", "vlm_output": "$steps.claude.output", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_csv_formatter.py b/tests/workflows/integration_tests/execution/test_workflow_with_csv_formatter.py index 8dd731d48..3bb13c19d 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_csv_formatter.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_csv_formatter.py @@ -19,7 +19,7 @@ ], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "model", "images": "$inputs.image", "model_id": "yolov8n-640", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_data_aggregation.py b/tests/workflows/integration_tests/execution/test_workflow_with_data_aggregation.py index 5f9ec4bb5..9d04783c9 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_data_aggregation.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_data_aggregation.py @@ -20,7 +20,7 @@ ], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "model", "images": "$inputs.image", "model_id": "$inputs.model_id", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_file_sink.py b/tests/workflows/integration_tests/execution/test_workflow_with_file_sink.py index 00c79d676..5f908782b 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_file_sink.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_file_sink.py @@ -26,7 +26,7 @@ ], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "model", "images": "$inputs.image", "model_id": "$inputs.model_id", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_florence2.py b/tests/workflows/integration_tests/execution/test_workflow_with_florence2.py index 35e3c9d93..055a8d289 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_florence2.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_florence2.py @@ -23,7 +23,7 @@ ], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "model_1", "images": "$inputs.image", "model_id": "yolov8n-640", @@ -148,7 +148,7 @@ def test_florence2_grounded_classification_when_no_grounding_available( "inputs": [{"type": "InferenceImage", "name": "image"}], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "model_1", "images": "$inputs.image", "model_id": "yolov8n-640", @@ -298,7 +298,7 @@ def test_florence2_instance_segmentation_grounded_by_input( "inputs": [{"type": "InferenceImage", "name": "image"}], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "model_1", "images": "$inputs.image", "model_id": "yolov8n-640", @@ -380,7 +380,7 @@ def test_florence2_grounded_caption( ), "Expected dog to be output by florence2" -FLORENCE_OBJECT_DETECTION_WORKFLOW = { +FLORENCE_OBJECT_DETECTION_WORKFLOW_LEGACY_PARSER = { "version": "1.0", "inputs": [ {"type": "InferenceImage", "name": "image"}, @@ -426,6 +426,90 @@ def test_florence2_grounded_caption( } +@pytest.mark.skipif( + bool_env(os.getenv("SKIP_FLORENCE2_TEST", True)), reason="Skipping Florence 2 test" +) +def test_florence2_object_detection_when_legacy_parser_is_in_use( + model_manager: ModelManager, + dogs_image: np.ndarray, + roboflow_api_key: str, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": roboflow_api_key, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=FLORENCE_OBJECT_DETECTION_WORKFLOW_LEGACY_PARSER, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={"image": dogs_image, "classes": ["dog"]} + ) + + assert isinstance(result, list), "Expected list to be delivered" + assert len(result) == 1, "Expected 1 element in the output for one input image" + assert set(result[0].keys()) == { + "predictions", + "bounding_box_visualization", + }, "Expected all declared outputs to be delivered" + assert len(result[0]["predictions"]) == 2, "Expected two predictions" + assert result[0]["predictions"].data["class_name"].tolist() == [ + "dog", + "dog", + ], "Expected two dogs to be found" + + +FLORENCE_OBJECT_DETECTION_WORKFLOW = { + "version": "1.0", + "inputs": [ + {"type": "InferenceImage", "name": "image"}, + {"type": "WorkflowParameter", "name": "classes"}, + ], + "steps": [ + { + "type": "roboflow_core/florence_2@v1", + "name": "model", + "images": "$inputs.image", + "task_type": "open-vocabulary-object-detection", + "classes": "$inputs.classes", + }, + { + "type": "roboflow_core/vlm_as_detector@v2", + "name": "vlm_as_detector", + "image": "$inputs.image", + "vlm_output": "$steps.model.raw_output", + "classes": "$steps.model.classes", + "model_type": "florence-2", + "task_type": "open-vocabulary-object-detection", + }, + { + "type": "roboflow_core/bounding_box_visualization@v1", + "name": "bounding_box_visualization", + "image": "$inputs.image", + "predictions": "$steps.vlm_as_detector.predictions", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "predictions", + "selector": "$steps.vlm_as_detector.predictions", + }, + { + "type": "JsonField", + "name": "bounding_box_visualization", + "coordinates_system": "own", + "selector": "$steps.bounding_box_visualization.image", + }, + ], +} + + @add_to_workflows_gallery( category="Workflows with Visual Language Models", use_case_title="Florence 2 - object detection", @@ -502,7 +586,7 @@ def test_florence2_object_detection( "task_type": "object-detection", }, { - "type": "roboflow_core/vlm_as_detector@v1", + "type": "roboflow_core/vlm_as_detector@v2", "name": "vlm_as_detector", "image": "$inputs.image", "vlm_output": "$steps.model.raw_output", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_gemini_models.py b/tests/workflows/integration_tests/execution/test_workflow_with_gemini_models.py index 0a583a6b4..5130f91a2 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_gemini_models.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_gemini_models.py @@ -302,7 +302,7 @@ def test_workflow_with_captioning_prompt( ), "Expected non-empty string generated" -CLASSIFICATION_WORKFLOW = { +CLASSIFICATION_WORKFLOW_LEGACY_PARSER = { "version": "1.0", "inputs": [ {"type": "WorkflowImage", "name": "image"}, @@ -354,12 +354,106 @@ def test_workflow_with_captioning_prompt( } +@pytest.mark.skipif( + condition=GOOGLE_API_KEY is None, reason="Google API key not provided" +) +def test_workflow_with_multi_class_classifier_prompt_and_legacy_parser( + model_manager: ModelManager, + dogs_image: np.ndarray, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=CLASSIFICATION_WORKFLOW_LEGACY_PARSER, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [dogs_image], + "api_key": GOOGLE_API_KEY, + "classes": ["cat", "dog"], + } + ) + + # then + assert len(result) == 1, "Single image given, expected single output" + assert set(result[0].keys()) == { + "gemini_result", + "top_class", + "parsed_prediction", + }, "Expected all outputs to be delivered" + assert ( + isinstance(result[0]["gemini_result"], str) + and len(result[0]["gemini_result"]) > 0 + ), "Expected non-empty string generated" + assert result[0]["top_class"] == "dog" + assert result[0]["parsed_prediction"]["error_status"] is False + + +CLASSIFICATION_WORKFLOW = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + {"type": "WorkflowParameter", "name": "api_key"}, + {"type": "WorkflowParameter", "name": "classes"}, + ], + "steps": [ + { + "type": "roboflow_core/google_gemini@v1", + "name": "gemini", + "images": "$inputs.image", + "task_type": "classification", + "classes": "$inputs.classes", + "api_key": "$inputs.api_key", + }, + { + "type": "roboflow_core/vlm_as_classifier@v2", + "name": "parser", + "image": "$inputs.image", + "vlm_output": "$steps.gemini.output", + "classes": "$steps.gemini.classes", + }, + { + "type": "roboflow_core/property_definition@v1", + "name": "top_class", + "operations": [ + {"type": "ClassificationPropertyExtract", "property_name": "top_class"} + ], + "data": "$steps.parser.predictions", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "gemini_result", + "selector": "$steps.gemini.output", + }, + { + "type": "JsonField", + "name": "top_class", + "selector": "$steps.top_class.output", + }, + { + "type": "JsonField", + "name": "parsed_prediction", + "selector": "$steps.parser.*", + }, + ], +} + + @add_to_workflows_gallery( category="Workflows with Visual Language Models", use_case_title="Using Google's Gemini as multi-class classifier", use_case_description=""" In this example, Google's Gemini model is used as classifier. Output from the model is parsed by -special `roboflow_core/vlm_as_classifier@v1` block which turns model output text into +special `roboflow_core/vlm_as_classifier@v2` block which turns model output text into full-blown prediction, which can later be used by other blocks compatible with classification predictions - in this case we extract top-class property. """, @@ -425,7 +519,7 @@ def test_workflow_with_multi_class_classifier_prompt( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$inputs.image", "vlm_output": "$steps.gemini.output", @@ -460,7 +554,7 @@ def test_workflow_with_multi_class_classifier_prompt( use_case_title="Using Google's Gemini as multi-label classifier", use_case_description=""" In this example, Google's Gemini model is used as multi-label classifier. Output from the model is parsed by -special `roboflow_core/vlm_as_classifier@v1` block which turns model output text into +special `roboflow_core/vlm_as_classifier@v2` block which turns model output text into full-blown prediction, which can later be used by other blocks compatible with classification predictions - in this case we extract top-class property. """, @@ -589,7 +683,7 @@ def test_workflow_with_structured_prompt( assert result[0]["result"] == "2" -OBJECT_DETECTION_WORKFLOW = { +OBJECT_DETECTION_WORKFLOW_LEGACY_PARSER = { "version": "1.0", "inputs": [ {"type": "WorkflowImage", "name": "image"}, @@ -630,6 +724,86 @@ def test_workflow_with_structured_prompt( } +@pytest.mark.skipif( + condition=GOOGLE_API_KEY is None, reason="Google API key not provided" +) +def test_workflow_with_object_detection_prompt_and_legacy_parser( + model_manager: ModelManager, + dogs_image: np.ndarray, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=OBJECT_DETECTION_WORKFLOW_LEGACY_PARSER, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [dogs_image], + "api_key": GOOGLE_API_KEY, + "classes": ["cat", "dog"], + } + ) + + # then + assert len(result) == 1, "Single image given, expected single output" + assert set(result[0].keys()) == { + "gemini_result", + "parsed_prediction", + }, "Expected all outputs to be delivered" + assert result[0]["parsed_prediction"].data["class_name"].tolist() == [ + "dog", + "dog", + ], "Expected 2 dogs to be detected" + + +OBJECT_DETECTION_WORKFLOW = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + {"type": "WorkflowParameter", "name": "api_key"}, + {"type": "WorkflowParameter", "name": "classes"}, + ], + "steps": [ + { + "type": "roboflow_core/google_gemini@v1", + "name": "gemini", + "images": "$inputs.image", + "task_type": "object-detection", + "classes": "$inputs.classes", + "api_key": "$inputs.api_key", + }, + { + "type": "roboflow_core/vlm_as_detector@v2", + "name": "parser", + "vlm_output": "$steps.gemini.output", + "image": "$inputs.image", + "classes": "$steps.gemini.classes", + "model_type": "google-gemini", + "task_type": "object-detection", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "gemini_result", + "selector": "$steps.gemini.output", + }, + { + "type": "JsonField", + "name": "parsed_prediction", + "selector": "$steps.parser.predictions", + }, + ], +} + + @add_to_workflows_gallery( category="Workflows with Visual Language Models", use_case_title="Using Google's Gemini as object-detection model", @@ -718,7 +892,7 @@ def test_workflow_with_object_detection_prompt( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$steps.cropping.crops", "vlm_output": "$steps.gemini.output", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_label_visualization.py b/tests/workflows/integration_tests/execution/test_workflow_with_label_visualization.py new file mode 100644 index 000000000..5584e962a --- /dev/null +++ b/tests/workflows/integration_tests/execution/test_workflow_with_label_visualization.py @@ -0,0 +1,73 @@ +import numpy as np + +from inference.core.env import WORKFLOWS_MAX_CONCURRENT_STEPS +from inference.core.managers.base import ModelManager +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.execution_engine.core import ExecutionEngine + +VISUALIZATION_WORKFLOW = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + { + "type": "WorkflowParameter", + "name": "model_id", + "default_value": "yolov8n-640", + }, + {"type": "WorkflowParameter", "name": "confidence", "default_value": 0.3}, + ], + "steps": [ + { + "type": "RoboflowObjectDetectionModel", + "name": "detection", + "image": "$inputs.image", + "model_id": "$inputs.model_id", + "confidence": "$inputs.confidence", + }, + { + "type": "roboflow_core/label_visualization@v1", + "name": "label_visualization", + "predictions": "$steps.detection.predictions", + "image": "$inputs.image", + "text": "Tracker Id", + }, + ], + "outputs": [ + {"type": "JsonField", "name": "result", "selector": "$steps.detection.*"}, + { + "type": "JsonField", + "name": "visualized", + "selector": "$steps.label_visualization.image", + }, + ], +} + + +def test_workflow_when_detections_are_not_present( + model_manager: ModelManager, + crowd_image: np.ndarray, +) -> None: + """This test covers bug in label annotator block.""" + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=VISUALIZATION_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={"image": crowd_image, "confidence": 0.99999} + ) + + # then + assert isinstance(result, list), "Expected result to be list" + assert len(result) == 1, "Single image provided - single output expected" + assert ( + len(result[0]["result"]["predictions"]) == 0 + ), "Expected no predictions to be delivered" diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_masked_crop.py b/tests/workflows/integration_tests/execution/test_workflow_with_masked_crop.py index 6e1684319..ccaa6337a 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_masked_crop.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_masked_crop.py @@ -8,7 +8,7 @@ add_to_workflows_gallery, ) -MASKED_CROP_WORKFLOW = { +MASKED_CROP_LEGACY_WORKFLOW = { "version": "1.0", "inputs": [ {"type": "WorkflowImage", "name": "image"}, @@ -54,6 +54,97 @@ } +def test_legacy_workflow_with_masked_crop( + model_manager: ModelManager, + dogs_image: np.ndarray, + roboflow_api_key: str, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": roboflow_api_key, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=MASKED_CROP_LEGACY_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": dogs_image, + } + ) + + assert isinstance(result, list), "Expected list to be delivered" + assert len(result) == 1, "Expected 1 element in the output for one input image" + assert set(result[0].keys()) == { + "crops", + "predictions", + }, "Expected all declared outputs to be delivered" + assert len(result[0]["crops"]) == 2, "Expected 2 crops for two dogs detected" + crop_image = result[0]["crops"][0].numpy_image + (x_min, y_min, x_max, y_max) = ( + result[0]["predictions"].xyxy[0].round().astype(dtype=int) + ) + crop_mask = result[0]["predictions"].mask[0][y_min:y_max, x_min:x_max] + pixels_outside_mask = np.where( + np.stack([crop_mask] * 3, axis=-1) == 0, + crop_image, + np.zeros_like(crop_image), + ) + pixels_sum = pixels_outside_mask.sum() + assert pixels_sum == 0, "Expected everything black outside mask" + + +MASKED_CROP_WORKFLOW = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + { + "type": "WorkflowParameter", + "name": "model_id", + "default_value": "yolov8n-seg-640", + }, + { + "type": "WorkflowParameter", + "name": "confidence", + "default_value": 0.4, + }, + ], + "steps": [ + { + "type": "roboflow_core/roboflow_instance_segmentation_model@v2", + "name": "segmentation", + "image": "$inputs.image", + "model_id": "$inputs.model_id", + "confidence": "$inputs.confidence", + }, + { + "type": "roboflow_core/dynamic_crop@v1", + "name": "cropping", + "image": "$inputs.image", + "predictions": "$steps.segmentation.predictions", + "mask_opacity": 1.0, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "crops", + "selector": "$steps.cropping.crops", + }, + { + "type": "JsonField", + "name": "predictions", + "selector": "$steps.segmentation.predictions", + }, + ], +} + + @add_to_workflows_gallery( category="Workflows with data transformations", use_case_title="Instance Segmentation results with background subtracted", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_model_comparision_visualisation.py b/tests/workflows/integration_tests/execution/test_workflow_with_model_comparision_visualisation.py index 9da2500a6..da1bf5d16 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_model_comparision_visualisation.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_model_comparision_visualisation.py @@ -26,13 +26,13 @@ ], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "model", "images": "$inputs.image", "model_id": "$inputs.model_1", }, { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "model_1", "images": "$inputs.image", "model_id": "$inputs.model_2", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_ocr_detections_stitching.py b/tests/workflows/integration_tests/execution/test_workflow_with_ocr_detections_stitching.py index b370602b3..5b625e35e 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_ocr_detections_stitching.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_ocr_detections_stitching.py @@ -22,7 +22,7 @@ ], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "ocr_detection", "image": "$inputs.image", "model_id": "$inputs.model_id", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_open_ai_models.py b/tests/workflows/integration_tests/execution/test_workflow_with_open_ai_models.py index 74074cccb..00b6e53ff 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_open_ai_models.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_open_ai_models.py @@ -303,7 +303,7 @@ def test_workflow_with_captioning_prompt( ), "Expected non-empty string generated" -CLASSIFICATION_WORKFLOW = { +CLASSIFICATION_WORKFLOW_WITH_LEGACY_PARSER = { "version": "1.0", "inputs": [ {"type": "WorkflowImage", "name": "image"}, @@ -355,12 +355,105 @@ def test_workflow_with_captioning_prompt( } +@pytest.mark.skipif( + condition=OPEN_AI_API_KEY is None, reason="OpenAI API key not provided" +) +def test_workflow_with_multi_class_classifier_prompt_and_legacy_parser( + model_manager: ModelManager, + dogs_image: np.ndarray, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=CLASSIFICATION_WORKFLOW_WITH_LEGACY_PARSER, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [dogs_image], + "api_key": OPEN_AI_API_KEY, + "classes": ["cat", "dog"], + } + ) + + # then + assert len(result) == 1, "Single image given, expected single output" + assert set(result[0].keys()) == { + "gpt_result", + "top_class", + "parsed_prediction", + }, "Expected all outputs to be delivered" + assert ( + isinstance(result[0]["gpt_result"], str) and len(result[0]["gpt_result"]) > 0 + ), "Expected non-empty string generated" + assert result[0]["top_class"] == "dog" + assert result[0]["parsed_prediction"]["error_status"] is False + + +CLASSIFICATION_WORKFLOW = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + {"type": "WorkflowParameter", "name": "api_key"}, + {"type": "WorkflowParameter", "name": "classes"}, + ], + "steps": [ + { + "type": "roboflow_core/open_ai@v2", + "name": "gpt", + "images": "$inputs.image", + "task_type": "classification", + "classes": "$inputs.classes", + "api_key": "$inputs.api_key", + }, + { + "type": "roboflow_core/vlm_as_classifier@v2", + "name": "parser", + "image": "$inputs.image", + "vlm_output": "$steps.gpt.output", + "classes": "$steps.gpt.classes", + }, + { + "type": "roboflow_core/property_definition@v1", + "name": "top_class", + "operations": [ + {"type": "ClassificationPropertyExtract", "property_name": "top_class"} + ], + "data": "$steps.parser.predictions", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "gpt_result", + "selector": "$steps.gpt.output", + }, + { + "type": "JsonField", + "name": "top_class", + "selector": "$steps.top_class.output", + }, + { + "type": "JsonField", + "name": "parsed_prediction", + "selector": "$steps.parser.*", + }, + ], +} + + @add_to_workflows_gallery( category="Workflows with Visual Language Models", use_case_title="Using GPT as multi-class classifier", use_case_description=""" In this example, GPT model is used as classifier. Output from the model is parsed by -special `roboflow_core/vlm_as_classifier@v1` block which turns GPT output text into +special `roboflow_core/vlm_as_classifier@v2` block which turns GPT output text into full-blown prediction, which can later be used by other blocks compatible with classification predictions - in this case we extract top-class property. """, @@ -425,7 +518,7 @@ def test_workflow_with_multi_class_classifier_prompt( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$inputs.image", "vlm_output": "$steps.gpt.output", @@ -627,7 +720,7 @@ def test_workflow_with_structured_prompt( "api_key": "$inputs.api_key", }, { - "type": "roboflow_core/vlm_as_classifier@v1", + "type": "roboflow_core/vlm_as_classifier@v2", "name": "parser", "image": "$steps.cropping.crops", "vlm_output": "$steps.gpt.output", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_rate_limiter.py b/tests/workflows/integration_tests/execution/test_workflow_with_rate_limiter.py index 13bc1965b..cd5b88e06 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_rate_limiter.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_rate_limiter.py @@ -15,7 +15,7 @@ ], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "model", "images": "$inputs.image", "model_id": "yolov8n-640", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_sahi.py b/tests/workflows/integration_tests/execution/test_workflow_with_sahi.py index bcdca5b99..f56a3f32b 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_sahi.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_sahi.py @@ -24,7 +24,7 @@ "image": "$inputs.image", }, { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "detection", "image": "$steps.image_slicer.slices", "model_id": "yolov8n-640", @@ -395,7 +395,7 @@ def slicer_callback(image_slice: np.ndarray): "image": "$inputs.image", }, { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "detection", "image": "$steps.image_slicer.slices", "model_id": "yolov8n-seg-640", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_sam2.py b/tests/workflows/integration_tests/execution/test_workflow_with_sam2.py index 9371b14c5..8116f7d5b 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_sam2.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_sam2.py @@ -179,7 +179,7 @@ def test_sam2_workflow_when_minimal_valid_input_provided_but_filtering_discard_m ], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "detection", "model_id": "yolov8n-640", "images": "$inputs.image", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_scalar_selectors.py b/tests/workflows/integration_tests/execution/test_workflow_with_scalar_selectors.py new file mode 100644 index 000000000..d330f482c --- /dev/null +++ b/tests/workflows/integration_tests/execution/test_workflow_with_scalar_selectors.py @@ -0,0 +1,234 @@ +from unittest import mock +from unittest.mock import MagicMock + +import numpy as np + +from inference.core.env import WORKFLOWS_MAX_CONCURRENT_STEPS +from inference.core.managers.base import ModelManager +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.execution_engine.core import ExecutionEngine +from inference.core.workflows.execution_engine.introspection import blocks_loader + +NON_BATCH_SECRET_STORE_WORKFLOW = { + "version": "1.3.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}], + "steps": [ + { + "type": "secret_store", + "name": "secret", + }, + { + "type": "secret_store_user", + "name": "user", + "image": "$inputs.image", + "secret": "$steps.secret.secret", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.user.output", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_with_scalar_selectors_for_batch_of_images( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, + dogs_image: np.ndarray, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.scalar_selectors_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=NON_BATCH_SECRET_STORE_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [dogs_image, dogs_image], + } + ) + + # then + assert len(result) == 2 + assert ( + result[0]["result"] == "my_secret" + ), "Expected secret store value propagated into output" + assert ( + result[1]["result"] == "my_secret" + ), "Expected secret store value propagated into output" + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_with_scalar_selectors_for_single_image( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, + dogs_image: np.ndarray, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.scalar_selectors_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=NON_BATCH_SECRET_STORE_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [dogs_image], + } + ) + + # then + assert len(result) == 1 + assert ( + result[0]["result"] == "my_secret" + ), "Expected secret store value propagated into output" + + +BATCH_SECRET_STORE_WORKFLOW = { + "version": "1.3.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}], + "steps": [ + { + "type": "batch_secret_store", + "name": "secret", + "image": "$inputs.image", + }, + { + "type": "non_batch_secret_store_user", + "name": "user", + "secret": "$steps.secret.secret", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.user.output", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_with_batch_oriented_secret_store_for_batch_of_images( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, + dogs_image: np.ndarray, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.scalar_selectors_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=BATCH_SECRET_STORE_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [dogs_image, dogs_image], + } + ) + + # then + assert len(result) == 2 + assert result[0]["result"].startswith( + "my_secret" + ), "Expected secret store value propagated into output" + assert result[1]["result"].startswith( + "my_secret" + ), "Expected secret store value propagated into output" + assert ( + result[0]["result"] != result[1]["result"] + ), "Expected different results for both outputs, as feature store should fire twice for two input images" + + +WORKFLOW_WITH_REFERENCE_SIMILARITY = { + "version": "1.3.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + {"type": "WorkflowParameter", "name": "reference"}, + ], + "steps": [ + { + "type": "reference_images_comparison", + "name": "comparison", + "image": "$inputs.image", + "reference_images": "$inputs.reference", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "similarity", + "selector": "$steps.comparison.similarity", + }, + ], +} + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_workflow_with_batch_oriented_secret_store_for_batch_of_images( + get_plugin_modules_mock: MagicMock, + model_manager: ModelManager, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.integration_tests.execution.stub_plugins.scalar_selectors_plugin", + ] + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + black_image = np.zeros((192, 168, 3), dtype=np.uint8) + red_image = np.ones((192, 168, 3), dtype=np.uint8) * (0, 0, 255) + white_image = (np.ones((192, 168, 3), dtype=np.uint8) * 255).astype(np.uint8) + execution_engine = ExecutionEngine.init( + workflow_definition=WORKFLOW_WITH_REFERENCE_SIMILARITY, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [black_image, red_image], + "reference": [black_image, red_image, white_image], + } + ) + + # then + assert len(result) == 2 + assert np.allclose(result[0]["similarity"], [1.0, 2 / 3, 0.0], atol=1e-2) + assert np.allclose(result[1]["similarity"], [2 / 3, 1.0, 1 / 3], atol=1e-2) diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_stitch_for_dynamic_crop.py b/tests/workflows/integration_tests/execution/test_workflow_with_stitch_for_dynamic_crop.py index 5edd32f5b..0b95a9006 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_stitch_for_dynamic_crop.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_stitch_for_dynamic_crop.py @@ -15,7 +15,7 @@ ], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "car_detection", "image": "$inputs.image", "model_id": "yolov8n-640", @@ -28,7 +28,7 @@ "predictions": "$steps.car_detection.predictions", }, { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "plates_detection", "image": "$steps.cropping.crops", "model_id": "vehicle-registration-plates-trudk/2", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_two_stage_models_and_flow_control.py b/tests/workflows/integration_tests/execution/test_workflow_with_two_stage_models_and_flow_control.py index ccf383ff7..721443c09 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_two_stage_models_and_flow_control.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_two_stage_models_and_flow_control.py @@ -10,7 +10,7 @@ "inputs": [{"type": "WorkflowImage", "name": "image"}], "steps": [ { - "type": "roboflow_core/roboflow_object_detection_model@v1", + "type": "roboflow_core/roboflow_object_detection_model@v2", "name": "general_detection", "image": "$inputs.image", "model_id": "yolov8n-640", @@ -23,7 +23,7 @@ "predictions": "$steps.general_detection.predictions", }, { - "type": "roboflow_core/roboflow_classification_model@v1", + "type": "roboflow_core/roboflow_classification_model@v2", "name": "breds_classification", "image": "$steps.cropping.crops", "model_id": "dog-breed-xpaq6/1", diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_video_metadata_processing.py b/tests/workflows/integration_tests/execution/test_workflow_with_video_metadata_processing.py index c50ea7116..fb5acca28 100644 --- a/tests/workflows/integration_tests/execution/test_workflow_with_video_metadata_processing.py +++ b/tests/workflows/integration_tests/execution/test_workflow_with_video_metadata_processing.py @@ -172,7 +172,7 @@ def test_workflow_with_tracker( "fps": 50, "comes_from_video_file": True, } - metadata_license_plare_image = { + metadata_license_plate_image = { "video_identifier": "c", "frame_number": 1, "frame_timestamp": datetime.now().isoformat(), @@ -197,7 +197,7 @@ def test_workflow_with_tracker( result_3 = execution_engine.run( runtime_parameters={ "image": [dogs_image, license_plate_image], - "video_metadata": [metadata_dogs_image, metadata_license_plare_image], + "video_metadata": [metadata_dogs_image, metadata_license_plate_image], } ) first_dogs_frame_tracker_ids = result_1[0]["tracker_id"] diff --git a/tests/workflows/unit_tests/core_steps/analytics/test_line_counter_v2.py b/tests/workflows/unit_tests/core_steps/analytics/test_line_counter_v2.py index b9f7d4396..eb6e8797b 100644 --- a/tests/workflows/unit_tests/core_steps/analytics/test_line_counter_v2.py +++ b/tests/workflows/unit_tests/core_steps/analytics/test_line_counter_v2.py @@ -63,8 +63,18 @@ def test_line_counter() -> None: ) # then - assert frame1_result == {"count_in": 0, "count_out": 0} - assert frame2_result == {"count_in": 1, "count_out": 1} + assert frame1_result == { + "count_in": 0, + "count_out": 0, + "detections_in": frame1_detections[[False, False, False, False]], + "detections_out": frame1_detections[[False, False, False, False]], + } + assert frame2_result == { + "count_in": 1, + "count_out": 1, + "detections_in": frame2_detections[[True, False, False, False]], + "detections_out": frame2_detections[[False, True, False, False]], + } def test_line_counter_no_trackers() -> None: diff --git a/tests/workflows/unit_tests/core_steps/common/test_deserializers.py b/tests/workflows/unit_tests/core_steps/common/test_deserializers.py new file mode 100644 index 000000000..d9e98c9a3 --- /dev/null +++ b/tests/workflows/unit_tests/core_steps/common/test_deserializers.py @@ -0,0 +1,683 @@ +import base64 + +import numpy as np +import pytest +import supervision as sv + +from inference.core.workflows.core_steps.common.deserializers import ( + deserialize_boolean_kind, + deserialize_bytes_kind, + deserialize_classification_prediction_kind, + deserialize_detections_kind, + deserialize_float_zero_to_one_kind, + deserialize_integer_kind, + deserialize_list_of_values_kind, + deserialize_numpy_array, + deserialize_optional_string_kind, + deserialize_point_kind, + deserialize_rgb_color_kind, + deserialize_zone_kind, +) +from inference.core.workflows.errors import RuntimeInputError + + +def test_deserialize_detections_kind_when_sv_detections_given() -> None: + # given + detections = sv.Detections.empty() + + # when + result = deserialize_detections_kind( + parameter="my_param", + detections=detections, + ) + + # then + assert result is detections, "Expected object not to be touched" + + +def test_deserialize_detections_kind_when_invalid_data_type_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_detections_kind( + parameter="my_param", + detections="INVALID", + ) + + +def test_deserialize_detections_kind_when_malformed_data_type_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_detections_kind( + parameter="my_param", + detections={ + "image": {"height": 100, "width": 300}, + # lack of predictions + }, + ) + + +def test_deserialize_detections_kind_when_serialized_empty_detections_given() -> None: + # given + detections = { + "image": {"height": 100, "width": 300}, + "predictions": [], + } + + # when + result = deserialize_detections_kind( + parameter="my_param", + detections=detections, + ) + + # then + assert isinstance(result, sv.Detections) + assert len(result) == 0 + + +def test_deserialize_detections_kind_when_serialized_non_empty_object_detections_given() -> ( + None +): + # given + detections = { + "image": { + "width": 168, + "height": 192, + }, + "predictions": [ + { + "data": "some", + "width": 1.0, + "height": 1.0, + "x": 1.5, + "y": 1.5, + "confidence": 0.1, + "class_id": 1, + "tracker_id": 1, + "class": "cat", + "detection_id": "first", + "parent_id": "image", + }, + ], + } + + # when + result = deserialize_detections_kind( + parameter="my_param", + detections=detections, + ) + + # then + assert isinstance(result, sv.Detections) + assert len(result) == 1 + assert np.allclose(result.xyxy, np.array([[1, 1, 2, 2]])) + assert result.data["class_name"] == np.array(["cat"]) + assert result.data["detection_id"] == np.array(["first"]) + assert result.data["parent_id"] == np.array(["image"]) + assert result.data["detection_id"] == np.array(["first"]) + assert np.allclose(result.data["image_dimensions"], np.array([[192, 168]])) + + +def test_deserialize_detections_kind_when_serialized_non_empty_instance_segmentations_given() -> ( + None +): + # given + detections = { + "image": { + "width": 168, + "height": 192, + }, + "predictions": [ + { + "data": "some", + "width": 1.0, + "height": 1.0, + "x": 1.5, + "y": 1.5, + "confidence": 0.1, + "class_id": 1, + "tracker_id": 1, + "class": "cat", + "detection_id": "first", + "parent_id": "image", + "points": [ + {"x": 1.0, "y": 1.0}, + {"x": 1.0, "y": 10.0}, + {"x": 10.0, "y": 10.0}, + {"x": 10.0, "y": 1.0}, + ], + }, + ], + } + + # when + result = deserialize_detections_kind( + parameter="my_param", + detections=detections, + ) + + # then + assert isinstance(result, sv.Detections) + assert len(result) == 1 + assert np.allclose(result.xyxy, np.array([[1, 1, 2, 2]])) + assert result.data["class_name"] == np.array(["cat"]) + assert result.data["detection_id"] == np.array(["first"]) + assert result.data["parent_id"] == np.array(["image"]) + assert result.data["detection_id"] == np.array(["first"]) + assert np.allclose(result.data["image_dimensions"], np.array([[192, 168]])) + assert result.mask.shape == (1, 192, 168) + + +def test_deserialize_detections_kind_when_serialized_non_empty_keypoints_detections_given() -> ( + None +): + # given + detections = { + "image": { + "width": 168, + "height": 192, + }, + "predictions": [ + { + "data": "some", + "width": 1.0, + "height": 1.0, + "x": 1.5, + "y": 1.5, + "confidence": 0.1, + "class_id": 1, + "tracker_id": 1, + "class": "cat", + "detection_id": "first", + "parent_id": "image", + "keypoints": [ + { + "class_id": 1, + "class_name": "nose", + "confidence": 0.1, + "x": 11.0, + "y": 11.0, + }, + { + "class_id": 2, + "class_name": "ear", + "confidence": 0.2, + "x": 12.0, + "y": 13.0, + }, + { + "class_id": 3, + "class_name": "eye", + "confidence": 0.3, + "x": 14.0, + "y": 15.0, + }, + ], + }, + ], + } + + # when + result = deserialize_detections_kind( + parameter="my_param", + detections=detections, + ) + + # then + assert isinstance(result, sv.Detections) + assert len(result) == 1 + assert np.allclose(result.xyxy, np.array([[1, 1, 2, 2]])) + assert result.data["class_name"] == np.array(["cat"]) + assert result.data["detection_id"] == np.array(["first"]) + assert result.data["parent_id"] == np.array(["image"]) + assert result.data["detection_id"] == np.array(["first"]) + assert np.allclose(result.data["image_dimensions"], np.array([[192, 168]])) + assert ( + result.data["keypoints_class_id"] + == np.array( + [np.array([1, 2, 3])], + dtype="object", + ) + ).all() + assert ( + result.data["keypoints_class_name"] + == np.array( + np.array(["nose", "ear", "eye"]), + dtype="object", + ) + ).all() + assert np.allclose( + result.data["keypoints_confidence"].astype(np.float64), + np.array([[0.1, 0.2, 0.3]], dtype=np.float64), + ) + + +def test_deserialize_numpy_array_when_numpy_array_is_given() -> None: + # given + raw_array = np.array([1, 2, 3]) + + # when + result = deserialize_numpy_array(parameter="some", raw_array=raw_array) + + # then + assert result is raw_array + + +def test_deserialize_numpy_array_when_serialized_array_is_given() -> None: + # given + raw_array = [1, 2, 3] + + # when + result = deserialize_numpy_array(parameter="some", raw_array=raw_array) + + # then + assert np.allclose(result, np.array([1, 2, 3])) + + +def test_deserialize_numpy_array_when_invalid_value_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_numpy_array(parameter="some", raw_array="invalid") + + +def test_deserialize_optional_string_kind_when_empty_value_given() -> None: + # when + result = deserialize_optional_string_kind(parameter="some", value=None) + + # then + assert result is None + + +def test_deserialize_optional_string_kind_when_string_given() -> None: + # when + result = deserialize_optional_string_kind(parameter="some", value="some") + + # then + assert result == "some" + + +def test_deserialize_optional_string_kind_when_invalid_value_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_optional_string_kind(parameter="some", value=b"some") + + +def test_deserialize_float_zero_to_one_kind_when_not_a_number_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_float_zero_to_one_kind(parameter="some", value="some") + + +def test_deserialize_float_zero_to_one_kind_when_integer_given() -> None: + # when + result = deserialize_float_zero_to_one_kind(parameter="some", value=1) + + # then + assert abs(result - 1.0) < 1e-5 + assert isinstance(result, float) + + +def test_deserialize_float_zero_to_one_kind_when_float_given() -> None: + # when + result = deserialize_float_zero_to_one_kind(parameter="some", value=0.5) + + # then + assert abs(result - 0.5) < 1e-5 + assert isinstance(result, float) + + +def test_deserialize_float_zero_to_one_kind_when_value_out_of_range_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_float_zero_to_one_kind(parameter="some", value=1.5) + + +def test_deserialize_list_of_values_kind_when_invalid_value_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_list_of_values_kind(parameter="some", value=1.5) + + +def test_deserialize_list_of_values_kind_when_list_given() -> None: + # when + result = deserialize_list_of_values_kind(parameter="some", value=[1, 2, 3]) + + # then + assert result == [1, 2, 3] + + +def test_deserialize_list_of_values_kind_when_tuple_given() -> None: + # when + result = deserialize_list_of_values_kind(parameter="some", value=(1, 2, 3)) + + # then + assert result == [1, 2, 3] + + +def test_deserialize_boolean_kind_when_boolean_given() -> None: + # when + result = deserialize_boolean_kind(parameter="some", value=True) + + # then + assert result is True + + +def test_deserialize_boolean_kind_when_invalid_value_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_boolean_kind(parameter="some", value="True") + + +def test_deserialize_integer_kind_when_integer_given() -> None: + # when + result = deserialize_integer_kind(parameter="some", value=3) + + # then + assert result == 3 + + +def test_deserialize_integer_kind_when_invalid_value_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_integer_kind(parameter="some", value=3.0) + + +def test_deserialize_classification_prediction_kind_when_valid_multi_class_prediction_given() -> ( + None +): + # given + prediction = { + "image": {"height": 128, "width": 256}, + "predictions": [{"class_name": "A", "class_id": 0, "confidence": 0.3}], + "top": "A", + "confidence": 0.3, + "parent_id": "some", + "prediction_type": "classification", + "inference_id": "some", + "root_parent_id": "some", + } + + # when + result = deserialize_classification_prediction_kind( + parameter="some", + value=prediction, + ) + + # then + assert result is prediction + + +def test_deserialize_classification_prediction_kind_when_valid_multi_label_prediction_given() -> ( + None +): + # given + prediction = { + "image": {"height": 128, "width": 256}, + "predictions": { + "a": {"confidence": 0.3, "class_id": 0}, + "b": {"confidence": 0.3, "class_id": 1}, + }, + "predicted_classes": ["a", "b"], + "parent_id": "some", + "prediction_type": "classification", + "inference_id": "some", + "root_parent_id": "some", + } + + # when + result = deserialize_classification_prediction_kind( + parameter="some", + value=prediction, + ) + + # then + assert result is prediction + + +def test_deserialize_classification_prediction_kind_when_not_a_dictionary_given() -> ( + None +): + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_classification_prediction_kind( + parameter="some", + value="invalid", + ) + + +@pytest.mark.parametrize( + "to_delete", + [ + ["image"], + ["predictions"], + ["top", "predicted_classes"], + ["confidence", "predicted_classes"], + ], +) +def test_deserialize_classification_prediction_kind_when_required_keys_not_given( + to_delete: list, +) -> None: + # given + prediction = { + "image": {"height": 128, "width": 256}, + "predictions": [{"class_name": "A", "class_id": 0, "confidence": 0.3}], + "top": "A", + "confidence": 0.3, + "predicted_classes": ["a", "b"], + "parent_id": "some", + "prediction_type": "classification", + "inference_id": "some", + "root_parent_id": "some", + } + for field in to_delete: + del prediction[field] + + with pytest.raises(RuntimeInputError): + _ = deserialize_classification_prediction_kind( + parameter="some", + value=prediction, + ) + + +def test_deserialize_zone_kind_when_valid_input_given() -> None: + # given + zone = [ + (1, 2), + [3, 4], + (5, 6), + ] + + # when + result = deserialize_zone_kind(parameter="some", value=zone) + + # then + assert result == [ + (1, 2), + [3, 4], + (5, 6), + ] + + +def test_deserialize_zone_kind_when_zone_misses_points() -> None: + # given + zone = [ + [3, 4], + (5, 6), + ] + + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_zone_kind(parameter="some", value=zone) + + +def test_deserialize_zone_kind_when_zone_has_invalid_elements() -> None: + # given + zone = [[3, 4], (5, 6), "invalid"] + + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_zone_kind(parameter="some", value=zone) + + +def test_deserialize_zone_kind_when_zone_defines_invalid_points() -> None: + # given + zone = [ + [3, 4], + (5, 6, 3), + (1, 2), + ] + + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_zone_kind(parameter="some", value=zone) + + +def test_deserialize_zone_kind_when_zone_defines_points_not_being_numbers() -> None: + # given + zone = [ + [3, 4], + (5, 6), + (1, "invalid"), + ] + + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_zone_kind(parameter="some", value=zone) + + +def test_deserialize_rgb_color_kind_when_invalid_value_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_rgb_color_kind(parameter="some", value=1) + + +def test_deserialize_rgb_color_kind_when_string_given() -> None: + # when + result = deserialize_rgb_color_kind(parameter="some", value="#fff") + + # then + assert result == "#fff" + + +def test_deserialize_rgb_color_kind_when_valid_tuple_given() -> None: + # when + result = deserialize_rgb_color_kind(parameter="some", value=(1, 2, 3)) + + # then + assert result == (1, 2, 3) + + +def test_deserialize_rgb_color_kind_when_to_short_tuple_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_rgb_color_kind(parameter="some", value=(1, 2)) + + +def test_deserialize_rgb_color_kind_when_to_long_tuple_given() -> None: + # when + result = deserialize_rgb_color_kind(parameter="some", value=(1, 2, 3, 4)) + + # then + assert result == (1, 2, 3) + + +def test_deserialize_rgb_color_kind_when_valid_list_given() -> None: + # when + result = deserialize_rgb_color_kind(parameter="some", value=[1, 2, 3]) + + # then + assert result == (1, 2, 3) + + +def test_deserialize_rgb_color_kind_when_to_short_list_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_rgb_color_kind(parameter="some", value=[1, 2]) + + +def test_deserialize_rgb_color_kind_when_to_long_list_given() -> None: + # when + result = deserialize_rgb_color_kind(parameter="some", value=[1, 2, 3, 4]) + + # then + assert result == (1, 2, 3) + + +def test_deserialize_point_kind_when_invalid_value_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_point_kind(parameter="some", value=1) + + +def test_deserialize_point_kind_when_valid_tuple_given() -> None: + # when + result = deserialize_point_kind(parameter="some", value=(1, 2)) + + # then + assert result == (1, 2) + + +def test_deserialize_point_kind_when_to_short_tuple_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_point_kind(parameter="some", value=(1,)) + + +def test_deserialize_point_kind_when_to_long_tuple_given() -> None: + # when + result = deserialize_point_kind(parameter="some", value=(1, 2, 3, 4)) + + # then + assert result == (1, 2) + + +def test_deserialize_point_kind_when_valid_list_given() -> None: + # when + result = deserialize_point_kind(parameter="some", value=[1, 2]) + + # then + assert result == (1, 2) + + +def test_deserialize_point_kind_when_to_short_list_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_point_kind(parameter="some", value=[1]) + + +def test_deserialize_point_kind_when_to_long_list_given() -> None: + # when + result = deserialize_point_kind(parameter="some", value=[1, 2, 3, 4]) + + # then + assert result == (1, 2) + + +def test_deserialize_point_kind_when_point_element_is_not_number() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_point_kind(parameter="some", value=[1, "invalid"]) + + +def test_deserialize_bytes_kind_when_invalid_value_given() -> None: + # when + with pytest.raises(RuntimeInputError): + _ = deserialize_bytes_kind(parameter="some", value=1) + + +def test_deserialize_bytes_kind_when_bytes_given() -> None: + # when + result = deserialize_bytes_kind(parameter="some", value=b"abcd") + + # then + assert result == b"abcd" + + +def test_deserialize_bytes_kind_when_base64_string_given() -> None: + # given + data = base64.b64encode(b"data").decode("utf-8") + + # when + result = deserialize_bytes_kind(parameter="some", value=data) + + # then + assert result == b"data" diff --git a/tests/workflows/unit_tests/core_steps/common/test_serializers.py b/tests/workflows/unit_tests/core_steps/common/test_serializers.py index af77d536c..219a513d6 100644 --- a/tests/workflows/unit_tests/core_steps/common/test_serializers.py +++ b/tests/workflows/unit_tests/core_steps/common/test_serializers.py @@ -7,6 +7,7 @@ from inference.core.workflows.core_steps.common.serializers import ( serialise_image, serialise_sv_detections, + serialize_wildcard_kind, ) from inference.core.workflows.execution_engine.entities.base import ( ImageParentMetadata, @@ -210,3 +211,148 @@ def test_serialise_image() -> None: assert ( recovered_image == np_image ).all(), "Recovered image should be equal to input image" + + +def test_serialize_wildcard_kind_when_workflow_image_data_is_given() -> None: + # given + np_image = np.zeros((192, 168, 3), dtype=np.uint8) + value = WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="some"), + numpy_image=np_image, + ) + + # when + result = serialize_wildcard_kind(value=value) + + # then + assert ( + result["type"] == "base64" + ), "Type of third element must be changed into base64" + decoded = base64.b64decode(result["value"]) + recovered_image = cv2.imdecode( + np.fromstring(decoded, dtype=np.uint8), + cv2.IMREAD_UNCHANGED, + ) + assert ( + recovered_image == np_image + ).all(), "Recovered image should be equal to input image" + + +def test_serialize_wildcard_kind_when_dictionary_is_given() -> None: + # given + np_image = np.zeros((192, 168, 3), dtype=np.uint8) + elements = { + "a": 3, + "b": "some", + "c": WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="some"), + numpy_image=np_image, + ), + } + + # when + result = serialize_wildcard_kind(value=elements) + + # then + assert len(result) == 3, "The same number of elements must be returned" + assert result["a"] == 3, "First element of list must be untouched" + assert result["b"] == "some", "Second element of list must be untouched" + assert ( + result["c"]["type"] == "base64" + ), "Type of third element must be changed into base64" + decoded = base64.b64decode(result["c"]["value"]) + recovered_image = cv2.imdecode( + np.fromstring(decoded, dtype=np.uint8), + cv2.IMREAD_UNCHANGED, + ) + assert ( + recovered_image == np_image + ).all(), "Recovered image should be equal to input image" + + +def test_serialize_wildcard_kind_when_list_is_given() -> None: + # given + np_image = np.zeros((192, 168, 3), dtype=np.uint8) + elements = [ + 3, + "some", + WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="some"), + numpy_image=np_image, + ), + ] + + # when + result = serialize_wildcard_kind(value=elements) + + # then + assert len(result) == 3, "The same number of elements must be returned" + assert result[0] == 3, "First element of list must be untouched" + assert result[1] == "some", "Second element of list must be untouched" + assert ( + result[2]["type"] == "base64" + ), "Type of third element must be changed into base64" + decoded = base64.b64decode(result[2]["value"]) + recovered_image = cv2.imdecode( + np.fromstring(decoded, dtype=np.uint8), + cv2.IMREAD_UNCHANGED, + ) + assert ( + recovered_image == np_image + ).all(), "Recovered image should be equal to input image" + + +def test_serialize_wildcard_kind_when_compound_input_is_given() -> None: + # given + np_image = np.zeros((192, 168, 3), dtype=np.uint8) + elements = [ + 3, + "some", + WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="some"), + numpy_image=np_image, + ), + { + "nested": [ + WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="other"), + numpy_image=np_image, + ) + ] + }, + ] + + # when + result = serialize_wildcard_kind(value=elements) + + # then + assert len(result) == 4, "The same number of elements must be returned" + assert result[0] == 3, "First element of list must be untouched" + assert result[1] == "some", "Second element of list must be untouched" + assert ( + result[2]["type"] == "base64" + ), "Type of third element must be changed into base64" + decoded = base64.b64decode(result[2]["value"]) + recovered_image = cv2.imdecode( + np.fromstring(decoded, dtype=np.uint8), + cv2.IMREAD_UNCHANGED, + ) + assert ( + recovered_image == np_image + ).all(), "Recovered image should be equal to input image" + nested_dict = result[3] + assert len(nested_dict["nested"]) == 1, "Expected one element in nested list" + assert ( + nested_dict["nested"][0]["type"] == "base64" + ), "Expected image serialized to base64" + assert ( + "video_metadata" in nested_dict["nested"][0] + ), "Expected video metadata attached" + decoded = base64.b64decode(nested_dict["nested"][0]["value"]) + recovered_image = cv2.imdecode( + np.fromstring(decoded, dtype=np.uint8), + cv2.IMREAD_UNCHANGED, + ) + assert ( + recovered_image == np_image + ).all(), "Recovered image should be equal to input image" diff --git a/tests/workflows/unit_tests/core_steps/formatters/vlm_as_classifier/__init__.py b/tests/workflows/unit_tests/core_steps/formatters/vlm_as_classifier/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/workflows/unit_tests/core_steps/formatters/test_vlm_as_classifier.py b/tests/workflows/unit_tests/core_steps/formatters/vlm_as_classifier/test_v1.py similarity index 100% rename from tests/workflows/unit_tests/core_steps/formatters/test_vlm_as_classifier.py rename to tests/workflows/unit_tests/core_steps/formatters/vlm_as_classifier/test_v1.py diff --git a/tests/workflows/unit_tests/core_steps/formatters/vlm_as_classifier/test_v2.py b/tests/workflows/unit_tests/core_steps/formatters/vlm_as_classifier/test_v2.py new file mode 100644 index 000000000..e22c744ce --- /dev/null +++ b/tests/workflows/unit_tests/core_steps/formatters/vlm_as_classifier/test_v2.py @@ -0,0 +1,342 @@ +from typing import List, Union + +import numpy as np +import pytest + +from inference.core.workflows.core_steps.formatters.vlm_as_classifier.v2 import ( + BlockManifest, + VLMAsClassifierBlockV2, +) +from inference.core.workflows.execution_engine.entities.base import ( + ImageParentMetadata, + WorkflowImageData, +) + + +@pytest.mark.parametrize("image", ["$inputs.image", "$steps.some.image"]) +@pytest.mark.parametrize( + "classes", ["$inputs.classes", "$steps.some.classes", ["a", "b"]] +) +def test_block_manifest_parsing_when_input_is_valid( + image: str, classes: Union[str, List[str]] +) -> None: + # given + raw_manifest = { + "type": "roboflow_core/vlm_as_classifier@v2", + "image": image, + "name": "parser", + "vlm_output": "$steps.vlm.output", + "classes": classes, + } + + # when + result = BlockManifest.model_validate(raw_manifest) + + # then + assert result == BlockManifest( + type="roboflow_core/vlm_as_classifier@v2", + name="parser", + image=image, + vlm_output="$steps.vlm.output", + classes=classes, + ) + + +def test_run_when_valid_json_given_for_multi_class_classification() -> None: + # given + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ +```json +{"class_name": "car", "confidence": "0.7"} +``` + """ + block = VLMAsClassifierBlockV2() + + # when + result = block.run(image=image, vlm_output=vlm_output, classes=["car", "cat"]) + + # then + assert result["error_status"] is False + assert result["predictions"]["image"] == {"width": 168, "height": 192} + assert result["predictions"]["predictions"] == [ + {"class": "car", "class_id": 0, "confidence": 0.7}, + {"class": "cat", "class_id": 1, "confidence": 0.0}, + ] + assert result["predictions"]["top"] == "car" + assert abs(result["predictions"]["confidence"] - 0.7) < 1e-5 + assert result["predictions"]["parent_id"] == "parent" + assert len(result["inference_id"]) > 0 + assert result["inference_id"] == result["predictions"]["inference_id"] + + +def test_run_when_valid_json_given_for_multi_class_classification_when_unknown_class_predicted() -> ( + None +): + # given + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ +```json +{"class_name": "my_class", "confidence": "0.7"} +``` + """ + block = VLMAsClassifierBlockV2() + + # when + result = block.run(image=image, vlm_output=vlm_output, classes=["car", "cat"]) + + # then + assert result["error_status"] is False + assert result["predictions"]["image"] == {"width": 168, "height": 192} + assert result["predictions"]["predictions"] == [ + {"class": "my_class", "class_id": -1, "confidence": 0.7}, + {"class": "car", "class_id": 0, "confidence": 0.0}, + {"class": "cat", "class_id": 1, "confidence": 0.0}, + ] + assert result["predictions"]["top"] == "my_class" + assert abs(result["predictions"]["confidence"] - 0.7) < 1e-5 + assert result["predictions"]["parent_id"] == "parent" + assert len(result["inference_id"]) > 0 + assert result["inference_id"] == result["predictions"]["inference_id"] + + +def test_run_when_valid_json_given_for_multi_label_classification() -> None: + # given + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ + {"predicted_classes": [ + {"class": "cat", "confidence": 0.3}, {"class": "dog", "confidence": 0.6}, + {"class": "cat", "confidence": "0.7"} + ]} + """ + block = VLMAsClassifierBlockV2() + + # when + result = block.run( + image=image, vlm_output=vlm_output, classes=["car", "cat", "dog"] + ) + + # then + assert result["error_status"] is False + assert result["predictions"]["image"] == {"width": 168, "height": 192} + assert result["predictions"]["predictions"] == { + "car": {"confidence": 0.0, "class_id": 0}, + "cat": {"confidence": 0.7, "class_id": 1}, + "dog": {"confidence": 0.6, "class_id": 2}, + } + assert set(result["predictions"]["predicted_classes"]) == {"cat", "dog"} + assert result["predictions"]["parent_id"] == "parent" + assert len(result["inference_id"]) > 0 + assert result["inference_id"] == result["predictions"]["inference_id"] + + +def test_run_when_valid_json_given_for_multi_label_classification_when_unknown_class_provided() -> ( + None +): + # given + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ + {"predicted_classes": [ + {"class": "my_class_1", "confidence": 0.3}, {"class": "my_class_2", "confidence": 0.6}, + {"class": "my_class_1", "confidence": 0.7} + ]} + """ + block = VLMAsClassifierBlockV2() + + # when + result = block.run( + image=image, vlm_output=vlm_output, classes=["car", "cat", "dog"] + ) + + # then + assert result["error_status"] is False + assert result["predictions"]["image"] == {"width": 168, "height": 192} + assert result["predictions"]["predictions"] == { + "car": {"confidence": 0.0, "class_id": 0}, + "cat": {"confidence": 0.0, "class_id": 1}, + "dog": {"confidence": 0.0, "class_id": 2}, + "my_class_1": {"confidence": 0.7, "class_id": -1}, + "my_class_2": {"confidence": 0.6, "class_id": -1}, + } + assert set(result["predictions"]["predicted_classes"]) == { + "my_class_1", + "my_class_2", + } + assert result["predictions"]["parent_id"] == "parent" + assert len(result["inference_id"]) > 0 + assert result["inference_id"] == result["predictions"]["inference_id"] + + +def test_run_when_valid_json_of_unknown_structure_given() -> None: + # given + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + block = VLMAsClassifierBlockV2() + + # when + result = block.run( + image=image, vlm_output='{"some": "data"}', classes=["car", "cat"] + ) + + # then + assert result["error_status"] is True + assert result["predictions"] is None + assert len(result["inference_id"]) > 0 + + +def test_run_when_invalid_json_given() -> None: + # given + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + block = VLMAsClassifierBlockV2() + + # when + result = block.run(image=image, vlm_output="invalid_json", classes=["car", "cat"]) + + # then + assert result["error_status"] is True + assert result["predictions"] is None + assert len(result["inference_id"]) > 0 + + +def test_run_when_multiple_jsons_given() -> None: + # given + raw_json = """ + {"predicted_classes": [ + {"class": "cat", "confidence": 0.3}, {"class": "dog", "confidence": 0.6}, + {"class": "cat", "confidence": "0.7"} + ]} + {"predicted_classes": [ + {"class": "cat", "confidence": 0.4}, {"class": "dog", "confidence": 0.7}, + {"class": "cat", "confidence": "0.8"} + ]} + """ + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + block = VLMAsClassifierBlockV2() + + # when + result = block.run(image=image, vlm_output=raw_json, classes=["car", "cat"]) + + # then + assert result["error_status"] is True + assert result["predictions"] is None + assert len(result["inference_id"]) > 0 + + +def test_run_when_json_in_markdown_block_given() -> None: + # given + raw_json = """ +```json +{"predicted_classes": [ + {"class": "cat", "confidence": 0.3}, {"class": "dog", "confidence": 0.6}, + {"class": "cat", "confidence": "0.7"} +]} +``` +``` + """ + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + block = VLMAsClassifierBlockV2() + + # when + result = block.run(image=image, vlm_output=raw_json, classes=["car", "cat", "dog"]) + + # then + assert result["error_status"] is False + assert result["predictions"]["image"] == {"width": 168, "height": 192} + assert result["predictions"]["predictions"] == { + "car": {"confidence": 0.0, "class_id": 0}, + "cat": {"confidence": 0.7, "class_id": 1}, + "dog": {"confidence": 0.6, "class_id": 2}, + } + assert set(result["predictions"]["predicted_classes"]) == {"cat", "dog"} + assert result["predictions"]["parent_id"] == "parent" + assert len(result["inference_id"]) > 0 + assert result["inference_id"] == result["predictions"]["inference_id"] + + +def test_run_when_json_in_markdown_block_without_new_lines_given() -> None: + # given + raw_json = """ +```json{"predicted_classes": [{"class": "cat", "confidence": 0.3}, {"class": "dog", "confidence": 0.6}, {"class": "cat", "confidence": "0.7"}]}``` +""" + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + block = VLMAsClassifierBlockV2() + + # when + result = block.run(image=image, vlm_output=raw_json, classes=["car", "cat", "dog"]) + + # then + assert result["error_status"] is False + assert result["predictions"]["image"] == {"width": 168, "height": 192} + assert result["predictions"]["predictions"] == { + "car": {"confidence": 0.0, "class_id": 0}, + "cat": {"confidence": 0.7, "class_id": 1}, + "dog": {"confidence": 0.6, "class_id": 2}, + } + assert set(result["predictions"]["predicted_classes"]) == {"cat", "dog"} + assert result["predictions"]["parent_id"] == "parent" + assert len(result["inference_id"]) > 0 + assert result["inference_id"] == result["predictions"]["inference_id"] + + +def test_run_when_multiple_jsons_in_markdown_block_given() -> None: + # given + raw_json = """ +```json +{"predicted_classes": [ + {"class": "cat", "confidence": 0.3}, {"class": "dog", "confidence": 0.6}, + {"class": "cat", "confidence": "0.7"} +]} +``` +```json +{"predicted_classes": [ + {"class": "cat", "confidence": 0.4}, {"class": "dog", "confidence": 0.7}, + {"class": "cat", "confidence": "0.8"} +]} +``` +""" + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + block = VLMAsClassifierBlockV2() + + # when + result = block.run(image=image, vlm_output=raw_json, classes=["car", "cat", "dog"]) + + # then + assert result["error_status"] is False + assert result["predictions"]["image"] == {"width": 168, "height": 192} + assert result["predictions"]["predictions"] == { + "car": {"confidence": 0.0, "class_id": 0}, + "cat": {"confidence": 0.7, "class_id": 1}, + "dog": {"confidence": 0.6, "class_id": 2}, + } + assert set(result["predictions"]["predicted_classes"]) == {"cat", "dog"} + assert result["predictions"]["parent_id"] == "parent" + assert len(result["inference_id"]) > 0 + assert result["inference_id"] == result["predictions"]["inference_id"] diff --git a/tests/workflows/unit_tests/core_steps/formatters/vlm_as_detector/__init__.py b/tests/workflows/unit_tests/core_steps/formatters/vlm_as_detector/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/workflows/unit_tests/core_steps/formatters/test_vlm_as_detector.py b/tests/workflows/unit_tests/core_steps/formatters/vlm_as_detector/test_v1.py similarity index 100% rename from tests/workflows/unit_tests/core_steps/formatters/test_vlm_as_detector.py rename to tests/workflows/unit_tests/core_steps/formatters/vlm_as_detector/test_v1.py diff --git a/tests/workflows/unit_tests/core_steps/formatters/vlm_as_detector/test_v2.py b/tests/workflows/unit_tests/core_steps/formatters/vlm_as_detector/test_v2.py new file mode 100644 index 000000000..3af0c5c10 --- /dev/null +++ b/tests/workflows/unit_tests/core_steps/formatters/vlm_as_detector/test_v2.py @@ -0,0 +1,404 @@ +from typing import List, Union + +import numpy as np +import pytest +import supervision as sv + +from inference.core.workflows.core_steps.formatters.vlm_as_detector.v2 import ( + BlockManifest, + VLMAsDetectorBlockV2, +) +from inference.core.workflows.execution_engine.entities.base import ( + ImageParentMetadata, + WorkflowImageData, +) + + +@pytest.mark.parametrize("image", ["$inputs.image", "$steps.some.image"]) +@pytest.mark.parametrize( + "classes", ["$inputs.classes", "$steps.some.classes", ["a", "b"]] +) +def test_manifest_parsing_when_input_valid( + image: str, classes: Union[str, List[str]] +) -> None: + # given + raw_manifest = { + "type": "roboflow_core/vlm_as_detector@v2", + "name": "parser", + "image": image, + "vlm_output": "$steps.vlm.output", + "classes": classes, + "model_type": "google-gemini", + "task_type": "object-detection", + } + + # when + result = BlockManifest.model_validate(raw_manifest) + + # then + assert result == BlockManifest( + type="roboflow_core/vlm_as_detector@v2", + name="parser", + image=image, + vlm_output="$steps.vlm.output", + classes=classes, + model_type="google-gemini", + task_type="object-detection", + ) + + +def test_run_method_for_claude_and_gemini_output() -> None: + # given + block = VLMAsDetectorBlockV2() + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ +{"detections": [ + {"x_min": 0.01, "y_min": 0.15, "x_max": 0.15, "y_max": 0.85, "class_name": "cat", "confidence": 1.98}, + {"x_min": 0.17, "y_min": 0.25, "x_max": 0.32, "y_max": 0.85, "class_name": "dog", "confidence": 0.97}, + {"x_min": 0.33, "y_min": 0.15, "x_max": 0.47, "y_max": 0.85, "class_name": "cat", "confidence": 0.99}, + {"x_min": 0.49, "y_min": 0.30, "x_max": 0.65, "y_max": 0.85, "class_name": "dog", "confidence": 0.98}, + {"x_min": 0.67, "y_min": 0.20, "x_max": 0.82, "y_max": 0.85, "class_name": "cat", "confidence": 0.99}, + {"x_min": 0.84, "y_min": 0.25, "x_max": 0.99, "y_max": 0.85, "class_name": "unknown", "confidence": 0.97} +]} + """ + + # when + result = block.run( + image=image, + vlm_output=vlm_output, + classes=["cat", "dog", "lion"], + model_type="google-gemini", + task_type="object-detection", + ) + + # then + assert result["error_status"] is False + assert isinstance(result["predictions"], sv.Detections) + assert len(result["inference_id"]) > 0 + assert np.allclose( + result["predictions"].xyxy, + np.array( + [ + [2, 29, 25, 163], + [29, 48, 54, 163], + [55, 29, 79, 163], + [82, 58, 109, 163], + [113, 38, 138, 163], + [141, 48, 166, 163], + ] + ), + atol=1.0, + ) + assert np.allclose(result["predictions"].class_id, np.array([0, 1, 0, 1, 0, -1])) + assert np.allclose( + result["predictions"].confidence, np.array([1.0, 0.97, 0.99, 0.98, 0.99, 0.97]) + ) + assert "class_name" in result["predictions"].data + assert "image_dimensions" in result["predictions"].data + assert "prediction_type" in result["predictions"].data + assert "parent_coordinates" in result["predictions"].data + assert "parent_dimensions" in result["predictions"].data + assert "root_parent_coordinates" in result["predictions"].data + assert "root_parent_dimensions" in result["predictions"].data + assert "parent_id" in result["predictions"].data + assert "root_parent_id" in result["predictions"].data + + +def test_run_method_for_invalid_claude_and_gemini_output() -> None: + # given + block = VLMAsDetectorBlockV2() + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ + {"detections": [ + {"x_min": 0.01, "y_min": 0.15, "x_max": 0.15, "y_max": 0.85, "confidence": 1.98}, + {"x_min": 0.17, "y_min": 0.25, "x_max": 0.32, "y_max": 0.85, "class_name": "dog", "confidence": 0.97}, + {"x_min": 0.33, "y_min": 0.15, "x_max": 0.47, "y_max": 0.85, "class_name": "cat", "confidence": 0.99}, + {"x_min": 0.49, "x_max": 0.65, "y_max": 0.85, "class_name": "dog", "confidence": 0.98}, + {"x_min": 0.67, "y_min": 0.20, "x_max": 0.82, "y_max": 0.85, "class_name": "cat", "confidence": 0.99}, + {"x_min": 0.84, "y_min": 0.25, "x_max": 0.99, "y_max": 0.85, "class_name": "unknown", "confidence": 0.97} + ]} + """ + + # when + result = block.run( + image=image, + vlm_output=vlm_output, + classes=["cat", "dog", "lion"], + model_type="google-gemini", + task_type="object-detection", + ) + + # then + assert result["error_status"] is True + assert result["predictions"] is None + assert len(result["inference_id"]) > 0 + + +def test_run_method_for_invalid_json() -> None: + # given + block = VLMAsDetectorBlockV2() + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + + # when + result = block.run( + image=image, + vlm_output="invalid", + classes=["cat", "dog", "lion"], + model_type="google-gemini", + task_type="object-detection", + ) + + # then + assert result["error_status"] is True + assert result["predictions"] is None + assert len(result["inference_id"]) > 0 + + +def test_formatter_for_florence2_object_detection() -> None: + # given + block = VLMAsDetectorBlockV2() + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ +{"bboxes": [[434.0, 30.848499298095703, 760.4000244140625, 530.4144897460938], [0.4000000059604645, 96.13949584960938, 528.4000244140625, 564.5574951171875]], "labels": ["cat", "dog"]} +""" + + # when + result = block.run( + image=image, + vlm_output=vlm_output, + classes=["cat", "dog"], + model_type="florence-2", + task_type="object-detection", + ) + + # then + assert result["error_status"] is False + assert isinstance(result["predictions"], sv.Detections) + assert len(result["inference_id"]) > 0 + assert np.allclose( + result["predictions"].xyxy, + np.array([[434, 30.848, 760.4, 530.41], [0.4, 96.139, 528.4, 564.56]]), + atol=1e-1, + ), "Expected coordinates to be the same as given in raw input" + assert result["predictions"].class_id.tolist() == [7725, 5324] + assert np.allclose(result["predictions"].confidence, np.array([1.0, 1.0])) + assert result["predictions"].data["class_name"].tolist() == ["cat", "dog"] + assert "class_name" in result["predictions"].data + assert "image_dimensions" in result["predictions"].data + assert "prediction_type" in result["predictions"].data + assert "parent_coordinates" in result["predictions"].data + assert "parent_dimensions" in result["predictions"].data + assert "root_parent_coordinates" in result["predictions"].data + assert "root_parent_dimensions" in result["predictions"].data + assert "parent_id" in result["predictions"].data + assert "root_parent_id" in result["predictions"].data + + +def test_formatter_for_florence2_open_vocabulary_object_detection() -> None: + # given + block = VLMAsDetectorBlockV2() + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ +{"bboxes": [[434.0, 30.848499298095703, 760.4000244140625, 530.4144897460938], [0.4000000059604645, 96.13949584960938, 528.4000244140625, 564.5574951171875]], "bboxes_labels": ["cat", "dog"]} +""" + + # when + result = block.run( + image=image, + vlm_output=vlm_output, + classes=["cat", "dog"], + model_type="florence-2", + task_type="open-vocabulary-object-detection", + ) + + # then + assert result["error_status"] is False + assert isinstance(result["predictions"], sv.Detections) + assert len(result["inference_id"]) > 0 + assert np.allclose( + result["predictions"].xyxy, + np.array([[434, 30.848, 760.4, 530.41], [0.4, 96.139, 528.4, 564.56]]), + atol=1e-1, + ), "Expected coordinates to be the same as given in raw input" + assert result["predictions"].class_id.tolist() == [0, 1] + assert np.allclose(result["predictions"].confidence, np.array([1.0, 1.0])) + assert result["predictions"].data["class_name"].tolist() == ["cat", "dog"] + assert "class_name" in result["predictions"].data + assert "image_dimensions" in result["predictions"].data + assert "prediction_type" in result["predictions"].data + assert "parent_coordinates" in result["predictions"].data + assert "parent_dimensions" in result["predictions"].data + assert "root_parent_coordinates" in result["predictions"].data + assert "root_parent_dimensions" in result["predictions"].data + assert "parent_id" in result["predictions"].data + assert "root_parent_id" in result["predictions"].data + + +def test_formatter_for_florence2_phase_grounded_detection() -> None: + # given + block = VLMAsDetectorBlockV2() + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ +{"bboxes": [[434.0, 30.848499298095703, 760.4000244140625, 530.4144897460938], [0.4000000059604645, 96.13949584960938, 528.4000244140625, 564.5574951171875]], "labels": ["cat", "dog"]} +""" + + # when + result = block.run( + image=image, + vlm_output=vlm_output, + classes=["cat", "dog"], + model_type="florence-2", + task_type="phrase-grounded-object-detection", + ) + + # then + assert result["error_status"] is False + assert isinstance(result["predictions"], sv.Detections) + assert len(result["inference_id"]) > 0 + assert np.allclose( + result["predictions"].xyxy, + np.array([[434, 30.848, 760.4, 530.41], [0.4, 96.139, 528.4, 564.56]]), + atol=1e-1, + ), "Expected coordinates to be the same as given in raw input" + assert result["predictions"].class_id.tolist() == [7725, 5324] + assert np.allclose(result["predictions"].confidence, np.array([1.0, 1.0])) + assert result["predictions"].data["class_name"].tolist() == ["cat", "dog"] + assert "class_name" in result["predictions"].data + assert "image_dimensions" in result["predictions"].data + assert "prediction_type" in result["predictions"].data + assert "parent_coordinates" in result["predictions"].data + assert "parent_dimensions" in result["predictions"].data + assert "root_parent_coordinates" in result["predictions"].data + assert "root_parent_dimensions" in result["predictions"].data + assert "parent_id" in result["predictions"].data + assert "root_parent_id" in result["predictions"].data + + +def test_formatter_for_florence2_region_proposal() -> None: + # given + block = VLMAsDetectorBlockV2() + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ +{"bboxes": [[434.0, 30.848499298095703, 760.4000244140625, 530.4144897460938], [0.4000000059604645, 96.13949584960938, 528.4000244140625, 564.5574951171875]], "labels": ["", ""]} +""" + + # when + result = block.run( + image=image, + vlm_output=vlm_output, + classes=[], + model_type="florence-2", + task_type="region-proposal", + ) + + # then + assert result["error_status"] is False + assert isinstance(result["predictions"], sv.Detections) + assert len(result["inference_id"]) > 0 + assert np.allclose( + result["predictions"].xyxy, + np.array([[434, 30.848, 760.4, 530.41], [0.4, 96.139, 528.4, 564.56]]), + atol=1e-1, + ), "Expected coordinates to be the same as given in raw input" + assert result["predictions"].class_id.tolist() == [0, 0] + assert np.allclose(result["predictions"].confidence, np.array([1.0, 1.0])) + assert result["predictions"].data["class_name"].tolist() == ["roi", "roi"] + assert "class_name" in result["predictions"].data + assert "image_dimensions" in result["predictions"].data + assert "prediction_type" in result["predictions"].data + assert "parent_coordinates" in result["predictions"].data + assert "parent_dimensions" in result["predictions"].data + assert "root_parent_coordinates" in result["predictions"].data + assert "root_parent_dimensions" in result["predictions"].data + assert "parent_id" in result["predictions"].data + assert "root_parent_id" in result["predictions"].data + + +def test_formatter_for_florence2_ocr() -> None: + # given + block = VLMAsDetectorBlockV2() + image = WorkflowImageData( + numpy_image=np.zeros((192, 168, 3), dtype=np.uint8), + parent_metadata=ImageParentMetadata(parent_id="parent"), + ) + vlm_output = """ +{"quad_boxes": [[336.9599914550781, 77.22000122070312, 770.8800048828125, 77.22000122070312, 770.8800048828125, 144.1800079345703, 336.9599914550781, 144.1800079345703], [1273.919921875, 77.22000122070312, 1473.5999755859375, 77.22000122070312, 1473.5999755859375, 109.62000274658203, 1273.919921875, 109.62000274658203], [1652.159912109375, 72.9000015258789, 1828.7999267578125, 70.74000549316406, 1828.7999267578125, 129.05999755859375, 1652.159912109375, 131.22000122070312], [1273.919921875, 126.9000015258789, 1467.8399658203125, 126.9000015258789, 1467.8399658203125, 160.3800048828125, 1273.919921875, 160.3800048828125], [340.79998779296875, 173.3400115966797, 964.7999877929688, 173.3400115966797, 964.7999877929688, 250.02000427246094, 340.79998779296875, 251.10000610351562], [1273.919921875, 177.66000366210938, 1473.5999755859375, 177.66000366210938, 1473.5999755859375, 208.98001098632812, 1273.919921875, 208.98001098632812], [1272.0, 226.260009765625, 1467.8399658203125, 226.260009765625, 1467.8399658203125, 259.7400207519531, 1272.0, 259.7400207519531], [340.79998779296875, 264.05999755859375, 801.5999755859375, 264.05999755859375, 801.5999755859375, 345.0600280761719, 340.79998779296875, 345.0600280761719], [1273.919921875, 277.02001953125, 1471.679931640625, 277.02001953125, 1471.679931640625, 309.4200134277344, 1273.919921875, 309.4200134277344], [1273.919921875, 326.70001220703125, 1467.8399658203125, 326.70001220703125, 1467.8399658203125, 359.1000061035156, 1273.919921875, 359.1000061035156], [336.9599914550781, 376.3800048828125, 980.1599731445312, 376.3800048828125, 980.1599731445312, 417.4200134277344, 336.9599914550781, 417.4200134277344]], "labels": ["What is OCR", "01010110", "veryfi", "010100101", "(Optical Character", "01010010", "011100101", "Recognition?", "0101010", "01010001", "A Friendly Introduction to OCR Software"]} +""" + + # when + result = block.run( + image=image, + vlm_output=vlm_output, + classes=[], + model_type="florence-2", + task_type="ocr-with-text-detection", + ) + + # then + assert result["error_status"] is False + assert isinstance(result["predictions"], sv.Detections) + assert len(result["inference_id"]) > 0 + assert np.allclose( + result["predictions"].xyxy, + np.array( + [ + [336.96, 77.22, 770.88, 144.18], + [1273.9, 77.22, 1473.6, 109.62], + [1652.2, 70.74, 1828.8, 131.22], + [1273.9, 126.9, 1467.8, 160.38], + [340.8, 173.34, 964.8, 251.1], + [1273.9, 177.66, 1473.6, 208.98], + [1272, 226.26, 1467.8, 259.74], + [340.8, 264.06, 801.6, 345.06], + [1273.9, 277.02, 1471.7, 309.42], + [1273.9, 326.7, 1467.8, 359.1], + [336.96, 376.38, 980.16, 417.42], + ] + ), + atol=1e-1, + ), "Expected coordinates to be the same as given in raw input" + assert result["predictions"].class_id.tolist() == [0] * 11 + assert np.allclose(result["predictions"].confidence, np.array([1.0] * 11)) + assert result["predictions"].data["class_name"].tolist() == [ + "What is OCR", + "01010110", + "veryfi", + "010100101", + "(Optical Character", + "01010010", + "011100101", + "Recognition?", + "0101010", + "01010001", + "A Friendly Introduction to OCR Software", + ] + assert "class_name" in result["predictions"].data + assert "image_dimensions" in result["predictions"].data + assert "prediction_type" in result["predictions"].data + assert "parent_coordinates" in result["predictions"].data + assert "parent_dimensions" in result["predictions"].data + assert "root_parent_coordinates" in result["predictions"].data + assert "root_parent_dimensions" in result["predictions"].data + assert "parent_id" in result["predictions"].data + assert "root_parent_id" in result["predictions"].data diff --git a/tests/workflows/unit_tests/core_steps/models/foundation/test_cogvlm.py b/tests/workflows/unit_tests/core_steps/models/foundation/test_cogvlm.py index 1e0dda23c..11472e8ae 100644 --- a/tests/workflows/unit_tests/core_steps/models/foundation/test_cogvlm.py +++ b/tests/workflows/unit_tests/core_steps/models/foundation/test_cogvlm.py @@ -297,7 +297,6 @@ def test_try_parse_cogvlm_output_to_json_when_multiple_json_markdown_blocks_with assert result == [{"field_a": 1, "field_b": 37}, {"field_a": 2, "field_b": 47}] -@mock.patch.object(v1, "WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS", 2) @mock.patch.object(v1, "WORKFLOWS_REMOTE_API_TARGET", "self-hosted") @mock.patch.object(v1.InferenceHTTPClient, "init") def test_get_cogvlm_generations_from_remote_api( diff --git a/tests/workflows/unit_tests/core_steps/models/foundation/test_lmm.py b/tests/workflows/unit_tests/core_steps/models/foundation/test_lmm.py index 561dc5299..2ccbf214c 100644 --- a/tests/workflows/unit_tests/core_steps/models/foundation/test_lmm.py +++ b/tests/workflows/unit_tests/core_steps/models/foundation/test_lmm.py @@ -396,7 +396,6 @@ def test_try_parse_lmm_output_to_json_when_multiple_json_markdown_blocks_with_mu assert result == [{"field_a": 1, "field_b": 37}, {"field_a": 2, "field_b": 47}] -@mock.patch.object(v1, "WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS", 2) @mock.patch.object(v1, "WORKFLOWS_REMOTE_API_TARGET", "self-hosted") @mock.patch.object(v1.InferenceHTTPClient, "init") def test_get_cogvlm_generations_from_remote_api( diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/instance_segmentation/__init__.py b/tests/workflows/unit_tests/core_steps/models/roboflow/instance_segmentation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/test_instance_segmentation.py b/tests/workflows/unit_tests/core_steps/models/roboflow/instance_segmentation/test_v1.py similarity index 100% rename from tests/workflows/unit_tests/core_steps/models/roboflow/test_instance_segmentation.py rename to tests/workflows/unit_tests/core_steps/models/roboflow/instance_segmentation/test_v1.py diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/instance_segmentation/test_v2.py b/tests/workflows/unit_tests/core_steps/models/roboflow/instance_segmentation/test_v2.py new file mode 100644 index 000000000..72edab06a --- /dev/null +++ b/tests/workflows/unit_tests/core_steps/models/roboflow/instance_segmentation/test_v2.py @@ -0,0 +1,133 @@ +from typing import Any + +import pytest +from pydantic import ValidationError + +from inference.core.workflows.core_steps.models.roboflow.instance_segmentation.v2 import ( + BlockManifest, +) + + +@pytest.mark.parametrize("images_field_alias", ["images", "image"]) +def test_instance_segmentation_model_validation_when_minimalistic_config_is_provided( + images_field_alias: str, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_instance_segmentation_model@v2", + "name": "some", + images_field_alias: "$inputs.image", + "model_id": "some/1", + } + + # when + result = BlockManifest.model_validate(data) + + # then + assert result == BlockManifest( + type="roboflow_core/roboflow_instance_segmentation_model@v2", + name="some", + images="$inputs.image", + model_id="some/1", + ) + + +@pytest.mark.parametrize("field", ["type", "name", "images", "model_id"]) +def test_instance_segmentation_model_validation_when_required_field_is_not_given( + field: str, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_instance_segmentation_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + } + del data[field] + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_instance_segmentation_model_validation_when_invalid_type_provided() -> None: + # given + data = { + "type": "invalid", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_instance_segmentation_model_validation_when_model_id_has_invalid_type() -> ( + None +): + # given + data = { + "type": "roboflow_core/roboflow_instance_segmentation_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": None, + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_instance_segmentation_model_validation_when_active_learning_flag_has_invalid_type() -> ( + None +): + # given + data = { + "type": "roboflow_core/roboflow_instance_segmentation_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + "disable_active_learning": "some", + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +@pytest.mark.parametrize( + "parameter, value", + [ + ("confidence", 1.1), + ("images", "some"), + ("disable_active_learning", "some"), + ("class_agnostic_nms", "some"), + ("class_filter", "some"), + ("confidence", "some"), + ("confidence", 1.1), + ("iou_threshold", "some"), + ("iou_threshold", 1.1), + ("max_detections", 0), + ("max_candidates", 0), + ("mask_decode_mode", "some"), + ("tradeoff_factor", 1.1), + ], +) +def test_instance_segmentation_model_when_parameters_have_invalid_type( + parameter: str, + value: Any, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_instance_segmentation_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + parameter: value, + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/keypoint_detection/__init__.py b/tests/workflows/unit_tests/core_steps/models/roboflow/keypoint_detection/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/test_keypoint_detection.py b/tests/workflows/unit_tests/core_steps/models/roboflow/keypoint_detection/test_v1.py similarity index 100% rename from tests/workflows/unit_tests/core_steps/models/roboflow/test_keypoint_detection.py rename to tests/workflows/unit_tests/core_steps/models/roboflow/keypoint_detection/test_v1.py diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/keypoint_detection/test_v2.py b/tests/workflows/unit_tests/core_steps/models/roboflow/keypoint_detection/test_v2.py new file mode 100644 index 000000000..f766781e1 --- /dev/null +++ b/tests/workflows/unit_tests/core_steps/models/roboflow/keypoint_detection/test_v2.py @@ -0,0 +1,132 @@ +from typing import Any + +import pytest +from pydantic import ValidationError + +from inference.core.workflows.core_steps.models.roboflow.keypoint_detection.v2 import ( + BlockManifest, +) + + +@pytest.mark.parametrize("images_field_alias", ["images", "image"]) +def test_keypoints_detection_model_validation_when_minimalistic_config_is_provided( + images_field_alias: str, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_keypoint_detection_model@v2", + "name": "some", + images_field_alias: "$inputs.image", + "model_id": "some/1", + } + + # when + result = BlockManifest.model_validate(data) + + # then + assert result == BlockManifest( + type="roboflow_core/roboflow_keypoint_detection_model@v2", + name="some", + images="$inputs.image", + model_id="some/1", + ) + + +@pytest.mark.parametrize("field", ["type", "name", "images", "model_id"]) +def test_keypoints_detection_model_validation_when_required_field_is_not_given( + field: str, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_keypoint_detection_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + } + del data[field] + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_keypoints_object_detection_model_validation_when_invalid_type_provided() -> ( + None +): + # given + data = { + "type": "invalid", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_keypoints_detection_model_validation_when_model_id_has_invalid_type() -> None: + # given + data = { + "type": "roboflow_core/roboflow_keypoint_detection_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": None, + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_keypoints_detection_model_validation_when_active_learning_flag_has_invalid_type() -> ( + None +): + # given + data = { + "type": "roboflow_core/roboflow_keypoint_detection_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + "disable_active_learning": "some", + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +@pytest.mark.parametrize( + "parameter, value", + [ + ("images", "some"), + ("disable_active_learning", "some"), + ("class_agnostic_nms", "some"), + ("class_filter", "some"), + ("confidence", "some"), + ("confidence", 1.1), + ("iou_threshold", "some"), + ("iou_threshold", 1.1), + ("max_detections", 0), + ("max_candidates", 0), + ("keypoint_confidence", "some"), + ("keypoint_confidence", 1.1), + ], +) +def test_keypoints_detection_model_when_parameters_have_invalid_type( + parameter: str, + value: Any, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_keypoint_detection_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + parameter: value, + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/multi_class_classification/__init__.py b/tests/workflows/unit_tests/core_steps/models/roboflow/multi_class_classification/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/test_multi_class_classification.py b/tests/workflows/unit_tests/core_steps/models/roboflow/multi_class_classification/test_v1.py similarity index 100% rename from tests/workflows/unit_tests/core_steps/models/roboflow/test_multi_class_classification.py rename to tests/workflows/unit_tests/core_steps/models/roboflow/multi_class_classification/test_v1.py diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/multi_class_classification/test_v2.py b/tests/workflows/unit_tests/core_steps/models/roboflow/multi_class_classification/test_v2.py new file mode 100644 index 000000000..5af8ac2f9 --- /dev/null +++ b/tests/workflows/unit_tests/core_steps/models/roboflow/multi_class_classification/test_v2.py @@ -0,0 +1,93 @@ +import pytest +from pydantic import ValidationError + +from inference.core.workflows.core_steps.models.roboflow.multi_class_classification.v2 import ( + BlockManifest, +) + + +@pytest.mark.parametrize("images_field_alias", ["images", "image"]) +def test_classification_model_validation_when_minimalistic_config_is_provided( + images_field_alias: str, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_classification_model@v2", + "name": "some", + images_field_alias: "$inputs.image", + "model_id": "some/1", + } + + # when + result = BlockManifest.model_validate(data) + + # then + assert result == BlockManifest( + type="roboflow_core/roboflow_classification_model@v2", + name="some", + images="$inputs.image", + model_id="some/1", + ) + + +@pytest.mark.parametrize("field", ["type", "name", "images", "model_id"]) +def test_classification_model_validation_when_required_field_is_not_given( + field: str, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_classification_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + } + del data[field] + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_classification_model_validation_when_invalid_type_provided() -> None: + # given + data = { + "type": "invalid", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_classification_model_validation_when_model_id_has_invalid_type() -> None: + # given + data = { + "type": "roboflow_core/roboflow_classification_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": None, + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_classification_model_validation_when_active_learning_flag_has_invalid_type() -> ( + None +): + # given + data = { + "type": "roboflow_core/roboflow_classification_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + "disable_active_learning": "some", + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/multi_label_classification/__init__.py b/tests/workflows/unit_tests/core_steps/models/roboflow/multi_label_classification/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/test_multi_label_classification.py b/tests/workflows/unit_tests/core_steps/models/roboflow/multi_label_classification/test_v1.py similarity index 100% rename from tests/workflows/unit_tests/core_steps/models/roboflow/test_multi_label_classification.py rename to tests/workflows/unit_tests/core_steps/models/roboflow/multi_label_classification/test_v1.py diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/multi_label_classification/test_v2.py b/tests/workflows/unit_tests/core_steps/models/roboflow/multi_label_classification/test_v2.py new file mode 100644 index 000000000..ec4acbd33 --- /dev/null +++ b/tests/workflows/unit_tests/core_steps/models/roboflow/multi_label_classification/test_v2.py @@ -0,0 +1,97 @@ +import pytest +from pydantic import ValidationError + +from inference.core.workflows.core_steps.models.roboflow.multi_label_classification.v2 import ( + BlockManifest, +) + + +@pytest.mark.parametrize("images_field_alias", ["images", "image"]) +def test_multi_label_classification_model_validation_when_minimalistic_config_is_provided( + images_field_alias: str, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_multi_label_classification_model@v2", + "name": "some", + images_field_alias: "$inputs.image", + "model_id": "some/1", + } + + # when + result = BlockManifest.model_validate(data) + + # then + assert result == BlockManifest( + type="roboflow_core/roboflow_multi_label_classification_model@v2", + name="some", + images="$inputs.image", + model_id="some/1", + ) + + +@pytest.mark.parametrize("field", ["type", "name", "images", "model_id"]) +def test_multi_label_classification_model_validation_when_required_field_is_not_given( + field: str, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_multi_label_classification_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + } + del data[field] + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_multi_label_classification_model_validation_when_invalid_type_provided() -> ( + None +): + # given + data = { + "type": "invalid", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_multi_label_classification_model_validation_when_model_id_has_invalid_type() -> ( + None +): + # given + data = { + "type": "roboflow_core/roboflow_multi_label_classification_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": None, + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) + + +def test_multi_label_classification_model_validation_when_active_learning_flag_has_invalid_type() -> ( + None +): + # given + data = { + "type": "roboflow_core/roboflow_multi_label_classification_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + "disable_active_learning": "some", + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(data) diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/object_detection/__init__.py b/tests/workflows/unit_tests/core_steps/models/roboflow/object_detection/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/test_object_detection.py b/tests/workflows/unit_tests/core_steps/models/roboflow/object_detection/test_v1.py similarity index 100% rename from tests/workflows/unit_tests/core_steps/models/roboflow/test_object_detection.py rename to tests/workflows/unit_tests/core_steps/models/roboflow/object_detection/test_v1.py diff --git a/tests/workflows/unit_tests/core_steps/models/roboflow/object_detection/test_v2.py b/tests/workflows/unit_tests/core_steps/models/roboflow/object_detection/test_v2.py new file mode 100644 index 000000000..a68ea8d03 --- /dev/null +++ b/tests/workflows/unit_tests/core_steps/models/roboflow/object_detection/test_v2.py @@ -0,0 +1,129 @@ +from typing import Any + +import pytest +from pydantic import ValidationError + +from inference.core.workflows.core_steps.models.roboflow.object_detection.v2 import ( + BlockManifest, +) + + +@pytest.mark.parametrize("images_field_alias", ["images", "image"]) +def test_object_detection_model_validation_when_minimalistic_config_is_provided( + images_field_alias: str, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_object_detection_model@v2", + "name": "some", + images_field_alias: "$inputs.image", + "model_id": "some/1", + } + + # when + result = BlockManifest.validate(data) + + # then + assert result == BlockManifest( + type="roboflow_core/roboflow_object_detection_model@v2", + name="some", + images="$inputs.image", + model_id="some/1", + ) + + +@pytest.mark.parametrize("field", ["type", "name", "images", "model_id"]) +def test_object_detection_model_validation_when_required_field_is_not_given( + field: str, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_object_detection_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + } + del data[field] + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.validate(data) + + +def test_object_detection_model_validation_when_invalid_type_provided() -> None: + # given + data = { + "type": "invalid", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.validate(data) + + +def test_object_detection_model_validation_when_model_id_has_invalid_type() -> None: + # given + data = { + "type": "roboflow_core/roboflow_object_detection_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": None, + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.validate(data) + + +def test_object_detection_model_validation_when_active_learning_flag_has_invalid_type() -> ( + None +): + # given + data = { + "type": "roboflow_core/roboflow_object_detection_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + "disable_active_learning": "some", + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.validate(data) + + +@pytest.mark.parametrize( + "parameter, value", + [ + ("confidence", 1.1), + ("images", "some"), + ("disable_active_learning", "some"), + ("class_agnostic_nms", "some"), + ("class_filter", "some"), + ("confidence", "some"), + ("confidence", 1.1), + ("iou_threshold", "some"), + ("iou_threshold", 1.1), + ("max_detections", 0), + ("max_candidates", 0), + ], +) +def test_object_detection_model_when_parameters_have_invalid_type( + parameter: str, + value: Any, +) -> None: + # given + data = { + "type": "roboflow_core/roboflow_object_detection_model@v2", + "name": "some", + "images": "$inputs.image", + "model_id": "some/1", + parameter: value, + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.validate(data) diff --git a/tests/workflows/unit_tests/core_steps/sinks/roboflow/test_roboflow_custom_metadata.py b/tests/workflows/unit_tests/core_steps/sinks/roboflow/test_roboflow_custom_metadata.py index b1807d6ab..1d1e4aa26 100644 --- a/tests/workflows/unit_tests/core_steps/sinks/roboflow/test_roboflow_custom_metadata.py +++ b/tests/workflows/unit_tests/core_steps/sinks/roboflow/test_roboflow_custom_metadata.py @@ -217,6 +217,79 @@ def test_run_when_fire_and_forget_with_background_tasks( assert len(background_tasks.tasks) == 1, "Expected background task to be added" +@patch( + "inference.core.workflows.core_steps.sinks.roboflow.custom_metadata.v1.add_custom_metadata_request" +) +def test_run_with_classification_results( + add_custom_metadata_request_mock: MagicMock, +) -> None: + # given + background_tasks = BackgroundTasks() + block = RoboflowCustomMetadataBlockV1( + cache=MemoryCache(), + api_key="my_api_key", + background_tasks=background_tasks, + thread_pool_executor=None, + ) + add_custom_metadata_request_mock.return_value = ( + False, + "Custom metadata upload was successful", + ) + predictions = {"inference_id": "some-id"} + + # when + result = block.run( + fire_and_forget=True, + field_name="location", + field_value="toronto", + predictions=predictions, + ) + + # then + assert result == { + "error_status": False, + "message": "Registration happens in the background task", + }, "Expected success message" + assert len(background_tasks.tasks) == 1, "Expected background task to be added" + + +@patch( + "inference.core.workflows.core_steps.sinks.roboflow.custom_metadata.v1.add_custom_metadata_request" +) +def test_run_with_classification_results_when_inference_id_is_not_given( + add_custom_metadata_request_mock: MagicMock, +) -> None: + # given + background_tasks = BackgroundTasks() + block = RoboflowCustomMetadataBlockV1( + cache=MemoryCache(), + api_key="my_api_key", + background_tasks=background_tasks, + thread_pool_executor=None, + ) + add_custom_metadata_request_mock.return_value = ( + False, + "Custom metadata upload was successful", + ) + predictions = {"predictions": ["a", "b", "c"]} + + # when + result = block.run( + fire_and_forget=True, + field_name="location", + field_value="toronto", + predictions=predictions, + ) + + # then + assert result == { + "error_status": True, + "message": "Custom metadata upload failed because no inference_ids were received. This is known bug " + "(https://github.com/roboflow/inference/issues/567). Please provide a report for the " + "problem under mentioned issue.", + }, "Expected failure due to no inference_ids" + + @patch( "inference.core.workflows.core_steps.sinks.roboflow.custom_metadata.v1.add_custom_metadata_request" ) diff --git a/tests/workflows/unit_tests/execution_engine/compiler/test_graph_constructor.py b/tests/workflows/unit_tests/execution_engine/compiler/test_graph_constructor.py index 8deafb0e0..3e12de6e9 100644 --- a/tests/workflows/unit_tests/execution_engine/compiler/test_graph_constructor.py +++ b/tests/workflows/unit_tests/execution_engine/compiler/test_graph_constructor.py @@ -12,6 +12,7 @@ ) from inference.core.workflows.execution_engine.entities.types import ( INTEGER_KIND, + OBJECT_DETECTION_PREDICTION_KIND, ROBOFLOW_MODEL_ID_KIND, ) from inference.core.workflows.execution_engine.v1.compiler.entities import ( @@ -122,6 +123,7 @@ def test_execution_graph_construction_for_trivial_workflow() -> None: selector="$outputs.predictions", data_lineage=[""], output_manifest=output_manifest, + kind=[OBJECT_DETECTION_PREDICTION_KIND], ), "Output node must be created correctly" assert result.has_edge( "$inputs.image", "$steps.model_1" diff --git a/tests/workflows/unit_tests/execution_engine/executor/test_output_constructor.py b/tests/workflows/unit_tests/execution_engine/executor/test_output_constructor.py index bb01cd8e5..7b452748a 100644 --- a/tests/workflows/unit_tests/execution_engine/executor/test_output_constructor.py +++ b/tests/workflows/unit_tests/execution_engine/executor/test_output_constructor.py @@ -6,7 +6,14 @@ import supervision as sv from networkx import DiGraph +from inference.core.workflows.core_steps.loader import KINDS_SERIALIZERS +from inference.core.workflows.errors import AssumptionError, ExecutionEngineRuntimeError from inference.core.workflows.execution_engine.entities.base import JsonField +from inference.core.workflows.execution_engine.entities.types import ( + IMAGE_KIND, + INTEGER_KIND, + STRING_KIND, +) from inference.core.workflows.execution_engine.v1.compiler.entities import ( NodeCategory, OutputNode, @@ -17,6 +24,7 @@ create_array, data_contains_sv_detections, place_data_in_array, + serialize_data_piece, ) @@ -413,6 +421,8 @@ def get_non_batch_data(selector: str) -> Any: workflow_outputs=workflow_outputs, execution_graph=execution_graph, execution_data_manager=execution_data_manager, + serialize_results=True, + kinds_serializers=KINDS_SERIALIZERS, ) # then @@ -529,6 +539,8 @@ def get_batch_data(selector: str, indices: List[tuple]) -> List[Any]: workflow_outputs=workflow_outputs, execution_graph=execution_graph, execution_data_manager=execution_data_manager, + serialize_results=True, + kinds_serializers=KINDS_SERIALIZERS, ) # then @@ -556,3 +568,190 @@ def get_batch_data(selector: str, indices: List[tuple]) -> List[Any]: "b_empty": None, "b_empty_nested": [[]], } + + +def test_serialize_data_piece_for_wildcard_output_when_serializer_not_found() -> None: + # when + result = serialize_data_piece( + output_name="my_output", + data_piece={"some": "data", "other": "another"}, + kind={"some": [STRING_KIND], "other": [STRING_KIND]}, + kinds_serializers={}, + ) + + # then + assert result == {"some": "data", "other": "another"}, "Expected data not t0 change" + + +def test_serialize_data_piece_for_wildcard_output_when_missmatch_in_input_detected() -> ( + None +): + # when + with pytest.raises(AssumptionError): + _ = serialize_data_piece( + output_name="my_output", + data_piece="not a dict", + kind={"some": [STRING_KIND], "other": [STRING_KIND]}, + kinds_serializers={}, + ) + + +def test_serialize_data_piece_for_wildcard_output_when_serializers_found_but_all_failing() -> ( + None +): + # given + def _faulty_serializer(value: Any) -> Any: + raise Exception() + + # when + with pytest.raises(ExecutionEngineRuntimeError): + _ = serialize_data_piece( + output_name="my_output", + data_piece={"some": "data", "other": "another"}, + kind={"some": [STRING_KIND, INTEGER_KIND], "other": STRING_KIND}, + kinds_serializers={ + STRING_KIND.name: _faulty_serializer, + INTEGER_KIND.name: _faulty_serializer, + }, + ) + + +def test_serialize_data_piece_for_wildcard_output_when_serializers_found_with_one_failing_and_one_successful() -> ( + None +): + # given + faulty_calls = [] + + def _faulty_serializer(value: Any) -> Any: + faulty_calls.append(1) + raise Exception() + + def _valid_serializer(value: Any) -> Any: + return "serialized", value + + # when + result = serialize_data_piece( + output_name="my_output", + data_piece={"some": "data", "other": "another"}, + kind={"some": [INTEGER_KIND, STRING_KIND], "other": [STRING_KIND]}, + kinds_serializers={ + STRING_KIND.name: _valid_serializer, + INTEGER_KIND.name: _faulty_serializer, + }, + ) + + # then + assert len(faulty_calls) == 1, "Expected faulty serializer attempted" + assert result == { + "some": ("serialized", "data"), + "other": ("serialized", "another"), + } + + +def test_serialize_data_piece_for_wildcard_output_when_serializers_found_and_successful() -> ( + None +): + # given + def _valid_serializer(value: Any) -> Any: + return "serialized", value + + # when + result = serialize_data_piece( + output_name="my_output", + data_piece={"some": "data", "other": "another"}, + kind={"some": [INTEGER_KIND, STRING_KIND], "other": [STRING_KIND]}, + kinds_serializers={ + STRING_KIND.name: _valid_serializer, + INTEGER_KIND.name: _valid_serializer, + }, + ) + + # then + assert result == { + "some": ("serialized", "data"), + "other": ("serialized", "another"), + } + + +def test_serialize_data_piece_for_specific_output_when_serializer_not_found() -> None: + # when + result = serialize_data_piece( + output_name="my_output", + data_piece="data", + kind=[STRING_KIND], + kinds_serializers={}, + ) + + # then + assert result == "data", "Expected data not to change" + + +def test_serialize_data_piece_for_specific_output_when_serializers_found_but_all_failing() -> ( + None +): + # given + def _faulty_serializer(value: Any) -> Any: + raise Exception() + + # when + with pytest.raises(ExecutionEngineRuntimeError): + _ = serialize_data_piece( + output_name="my_output", + data_piece="data", + kind=[STRING_KIND, INTEGER_KIND], + kinds_serializers={ + STRING_KIND.name: _faulty_serializer, + INTEGER_KIND.name: _faulty_serializer, + }, + ) + + +def test_serialize_data_piece_for_specific_output_when_serializers_found_with_one_failing_and_one_successful() -> ( + None +): + # given + faulty_calls = [] + + def _faulty_serializer(value: Any) -> Any: + faulty_calls.append(1) + raise Exception() + + def _valid_serializer(value: Any) -> Any: + return "serialized", value + + # when + result = serialize_data_piece( + output_name="my_output", + data_piece="data", + kind=[INTEGER_KIND, STRING_KIND], + kinds_serializers={ + STRING_KIND.name: _valid_serializer, + INTEGER_KIND.name: _faulty_serializer, + }, + ) + + # then + assert len(faulty_calls) == 1, "Expected faulty serializer attempted" + assert result == ("serialized", "data") + + +def test_serialize_data_piece_for_specific_output_when_serializers_found_and_successful() -> ( + None +): + # given + def _valid_serializer(value: Any) -> Any: + return "serialized", value + + # when + result = serialize_data_piece( + output_name="my_output", + data_piece="data", + kind=[INTEGER_KIND, STRING_KIND], + kinds_serializers={ + STRING_KIND.name: _valid_serializer, + INTEGER_KIND.name: _valid_serializer, + }, + ) + + # then + assert result == ("serialized", "data") diff --git a/tests/workflows/unit_tests/execution_engine/executor/test_runtime_input_assembler.py b/tests/workflows/unit_tests/execution_engine/executor/test_runtime_input_assembler.py index 9cac89a2a..b513be577 100644 --- a/tests/workflows/unit_tests/execution_engine/executor/test_runtime_input_assembler.py +++ b/tests/workflows/unit_tests/execution_engine/executor/test_runtime_input_assembler.py @@ -7,15 +7,25 @@ import numpy as np import pytest +from inference.core.workflows.core_steps.common import deserializers +from inference.core.workflows.core_steps.loader import KINDS_DESERIALIZERS from inference.core.workflows.errors import RuntimeInputError from inference.core.workflows.execution_engine.entities.base import ( VideoMetadata, + WorkflowBatchInput, WorkflowImage, + WorkflowImageData, WorkflowParameter, WorkflowVideoMetadata, ) -from inference.core.workflows.execution_engine.v1.executor import ( - runtime_input_assembler, +from inference.core.workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + DICTIONARY_KIND, + FLOAT_KIND, + IMAGE_KIND, + INTEGER_KIND, + LIST_OF_VALUES_KIND, + STRING_KIND, ) from inference.core.workflows.execution_engine.v1.executor.runtime_input_assembler import ( assemble_runtime_parameters, @@ -32,10 +42,11 @@ def test_assemble_runtime_parameters_when_image_is_not_provided() -> None: _ = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) -@mock.patch.object(runtime_input_assembler, "load_image_from_url") +@mock.patch.object(deserializers, "load_image_from_url") def test_assemble_runtime_parameters_when_image_is_provided_as_single_element_dict( load_image_from_url_mock: MagicMock, ) -> None: @@ -53,6 +64,7 @@ def test_assemble_runtime_parameters_when_image_is_provided_as_single_element_di result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -83,6 +95,7 @@ def test_assemble_runtime_parameters_when_image_is_provided_as_single_element_di result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -115,6 +128,7 @@ def test_assemble_runtime_parameters_when_image_is_provided_as_single_element_di runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, prevent_local_images_loading=True, + kinds_deserializers=KINDS_DESERIALIZERS, ) @@ -129,6 +143,7 @@ def test_assemble_runtime_parameters_when_image_is_provided_as_single_element_np result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -153,6 +168,7 @@ def test_assemble_runtime_parameters_when_image_is_provided_as_unknown_element() _ = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) @@ -173,6 +189,7 @@ def test_assemble_runtime_parameters_when_image_is_provided_in_batch() -> None: result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -225,6 +242,7 @@ def test_assemble_runtime_parameters_when_image_is_provided_with_video_metadata( result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -253,6 +271,7 @@ def test_assemble_runtime_parameters_when_parameter_not_provided() -> None: result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -268,6 +287,7 @@ def test_assemble_runtime_parameters_when_parameter_provided() -> None: result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -287,16 +307,19 @@ def test_assemble_runtime_parameters_when_images_with_different_matching_batch_s }, ], "image2": np.zeros((192, 168, 3), dtype=np.uint8), + "image3": [np.zeros((192, 168, 3), dtype=np.uint8)], } defined_inputs = [ WorkflowImage(type="WorkflowImage", name="image1"), WorkflowImage(type="WorkflowImage", name="image2"), + WorkflowImage(type="WorkflowImage", name="image3"), ] # when result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -312,6 +335,12 @@ def test_assemble_runtime_parameters_when_images_with_different_matching_batch_s assert np.allclose( result["image2"][1].numpy_image, np.zeros((192, 168, 3), dtype=np.uint8) ), "Empty image expected" + assert np.allclose( + result["image3"][0].numpy_image, np.zeros((192, 168, 3), dtype=np.uint8) + ), "Empty image expected" + assert np.allclose( + result["image3"][1].numpy_image, np.zeros((192, 168, 3), dtype=np.uint8) + ), "Empty image expected" def test_assemble_runtime_parameters_when_images_with_different_and_not_matching_batch_sizes_provided() -> ( @@ -338,6 +367,7 @@ def test_assemble_runtime_parameters_when_images_with_different_and_not_matching _ = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) @@ -379,6 +409,7 @@ def test_assemble_runtime_parameters_when_video_metadata_with_different_matching result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -405,6 +436,7 @@ def test_assemble_runtime_parameters_when_video_metadata_declared_but_not_provid _ = assemble_runtime_parameters( runtime_parameters={}, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) @@ -430,6 +462,7 @@ def test_assemble_runtime_parameters_when_video_metadata_declared_and_provided_a result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -456,6 +489,7 @@ def test_assemble_runtime_parameters_when_video_metadata_declared_and_provided_a result = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) # then @@ -505,4 +539,191 @@ def test_assemble_runtime_parameters_when_video_metadata_with_different_and_not_ _ = assemble_runtime_parameters( runtime_parameters=runtime_parameters, defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, + ) + + +def test_assemble_runtime_parameters_when_parameters_at_different_dimensionality_depth_emerge() -> ( + None +): + # given + runtime_parameters = { + "image1": [ + np.zeros((192, 168, 3), dtype=np.uint8), + np.zeros((192, 168, 3), dtype=np.uint8), + ], + "image2": [ + [ + np.zeros((192, 168, 3), dtype=np.uint8), + np.zeros((192, 168, 3), dtype=np.uint8), + ], + [ + np.zeros((192, 168, 3), dtype=np.uint8), + ], + ], + "image3": [ + [ + [np.zeros((192, 168, 3), dtype=np.uint8)], + [ + np.zeros((192, 168, 3), dtype=np.uint8), + np.zeros((192, 168, 3), dtype=np.uint8), + ], + ], + [ + [np.zeros((192, 168, 3), dtype=np.uint8)], + [ + np.zeros((192, 168, 3), dtype=np.uint8), + np.zeros((192, 168, 3), dtype=np.uint8), + ], + [np.zeros((192, 168, 3), dtype=np.uint8)], + ], + ], + } + defined_inputs = [ + WorkflowBatchInput(type="WorkflowBatchInput", name="image1", kind=["image"]), + WorkflowBatchInput( + type="WorkflowBatchInput", + name="image2", + kind=[IMAGE_KIND], + dimensionality=2, + ), + WorkflowBatchInput( + type="WorkflowBatchInput", name="image3", kind=["image"], dimensionality=3 + ), + ] + + # when + result = assemble_runtime_parameters( + runtime_parameters=runtime_parameters, + defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, + ) + + # then + assert len(result["image1"]) == 2, "image1 is 1D batch of size (2, )" + assert all( + isinstance(e, WorkflowImageData) for e in result["image1"] + ), "Expected deserialized image data at the bottom level of batch" + # then + sizes_of_image2 = [len(e) for e in result["image2"]] + assert sizes_of_image2 == [2, 1], "image1 is 2D batch of size [(2, ), (1, )]" + assert all( + isinstance(e, WorkflowImageData) + for nested_batch in result["image2"] + for e in nested_batch + ), "Expected deserialized image data at the bottom level of batch" + sizes_of_image3 = [ + [len(e) for e in inner_batch] for inner_batch in result["image3"] + ] + assert sizes_of_image3 == [ + [1, 2], + [1, 2, 1], + ], "image1 is 3D batch of size [[(1, ), (2, )], [(1, ), (2, ), (1, )]]" + assert all( + isinstance(e, WorkflowImageData) + for nested_batch in result["image3"] + for inner_batch in nested_batch + for e in inner_batch + ), "Expected deserialized image data at the bottom level of batch" + + +def test_assemble_runtime_parameters_when_basic_types_are_passed_as_batch_oriented_inputs() -> ( + None +): + # given + runtime_parameters = { + "string_param": ["a", "b"], + "float_param": [1.0, 2.0], + "int_param": [3, 4], + "list_param": [["some", "list"], ["other", "list"]], + "boolean_param": [False, True], + "dict_param": [{"some": "dict"}, {"other": "dict"}], + } + defined_inputs = [ + WorkflowBatchInput( + type="WorkflowBatchInput", name="string_param", kind=[STRING_KIND.name] + ), + WorkflowBatchInput( + type="WorkflowBatchInput", name="float_param", kind=[FLOAT_KIND.name] + ), + WorkflowBatchInput( + type="WorkflowBatchInput", name="int_param", kind=[INTEGER_KIND] + ), + WorkflowBatchInput( + type="WorkflowBatchInput", name="list_param", kind=[LIST_OF_VALUES_KIND] + ), + WorkflowBatchInput( + type="WorkflowBatchInput", name="boolean_param", kind=[BOOLEAN_KIND] + ), + WorkflowBatchInput( + type="WorkflowBatchInput", name="dict_param", kind=[DICTIONARY_KIND] + ), + ] + + # when + result = assemble_runtime_parameters( + runtime_parameters=runtime_parameters, + defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, + ) + + # then + assert result == { + "string_param": ["a", "b"], + "float_param": [1.0, 2.0], + "int_param": [3, 4], + "list_param": [["some", "list"], ["other", "list"]], + "boolean_param": [False, True], + "dict_param": [{"some": "dict"}, {"other": "dict"}], + }, "Expected values not to be changed" + + +def test_assemble_runtime_parameters_when_input_batch_shallower_than_declared() -> None: + # given + runtime_parameters = { + "string_param": ["a", "b"], + "float_param": [1.0, 2.0], + } + defined_inputs = [ + WorkflowBatchInput( + type="WorkflowBatchInput", name="string_param", kind=[STRING_KIND.name] + ), + WorkflowBatchInput( + type="WorkflowBatchInput", + name="float_param", + kind=[FLOAT_KIND.name], + dimensionality=2, + ), + ] + + # when + with pytest.raises(RuntimeInputError): + _ = assemble_runtime_parameters( + runtime_parameters=runtime_parameters, + defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, + ) + + +def test_assemble_runtime_parameters_when_input_batch_deeper_than_declared() -> None: + # given + runtime_parameters = { + "string_param": ["a", "b"], + "float_param": [[1.0], [2.0]], + } + defined_inputs = [ + WorkflowBatchInput( + type="WorkflowBatchInput", name="string_param", kind=[STRING_KIND.name] + ), + WorkflowBatchInput( + type="WorkflowBatchInput", name="float_param", kind=[FLOAT_KIND.name] + ), + ] + + # when + with pytest.raises(RuntimeInputError): + _ = assemble_runtime_parameters( + runtime_parameters=runtime_parameters, + defined_inputs=defined_inputs, + kinds_deserializers=KINDS_DESERIALIZERS, ) diff --git a/tests/workflows/unit_tests/execution_engine/introspection/plugin_with_kinds_serializers/__init__.py b/tests/workflows/unit_tests/execution_engine/introspection/plugin_with_kinds_serializers/__init__.py new file mode 100644 index 000000000..ae3a1b0fc --- /dev/null +++ b/tests/workflows/unit_tests/execution_engine/introspection/plugin_with_kinds_serializers/__init__.py @@ -0,0 +1,33 @@ +from typing import List, Type + +from inference.core.workflows.execution_engine.entities.types import Kind +from inference.core.workflows.prototypes.block import WorkflowBlock + +MY_KIND_1 = Kind(name="1") +MY_KIND_2 = Kind(name="2") +MY_KIND_3 = Kind(name="3") + + +def load_blocks() -> List[Type[WorkflowBlock]]: + return [] + + +def load_kinds() -> List[Kind]: + return [ + MY_KIND_1, + MY_KIND_2, + MY_KIND_3, + ] + + +KINDS_SERIALIZERS = { + "1": lambda value: "1", + "2": lambda value: "2", + "3": lambda value: "3", +} + +KINDS_DESERIALIZERS = { + "1": lambda name, value: "1", + "2": lambda name, value: "2", + "3": lambda name, value: "3", +} diff --git a/tests/workflows/unit_tests/execution_engine/introspection/test_blocks_loader.py b/tests/workflows/unit_tests/execution_engine/introspection/test_blocks_loader.py index ed12760fe..7de1b6553 100644 --- a/tests/workflows/unit_tests/execution_engine/introspection/test_blocks_loader.py +++ b/tests/workflows/unit_tests/execution_engine/introspection/test_blocks_loader.py @@ -19,6 +19,8 @@ load_blocks_from_plugin, load_initializers, load_initializers_from_plugin, + load_kinds_deserializers, + load_kinds_serializers, load_workflow_blocks, ) from tests.workflows.unit_tests.execution_engine.introspection import ( @@ -426,3 +428,47 @@ def test_is_block_compatible_with_execution_engine_when_block_execution_engine_c block_source="workflows_core", block_identifier="some", ) + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_load_kinds_serializers( + get_plugin_modules_mock: MagicMock, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.unit_tests.execution_engine.introspection.plugin_with_kinds_serializers" + ] + + # when + result = load_kinds_serializers() + + # then + assert len(result) > 0 + assert result["1"]("some") == "1", "Expected hardcoded value from serializer" + assert result["2"]("some") == "2", "Expected hardcoded value from serializer" + assert result["3"]("some") == "3", "Expected hardcoded value from serializer" + + +@mock.patch.object(blocks_loader, "get_plugin_modules") +def test_load_kinds_deserializers( + get_plugin_modules_mock: MagicMock, +) -> None: + # given + get_plugin_modules_mock.return_value = [ + "tests.workflows.unit_tests.execution_engine.introspection.plugin_with_kinds_serializers" + ] + + # when + result = load_kinds_deserializers() + + # then + assert len(result) > 0 + assert ( + result["1"]("some", "value") == "1" + ), "Expected hardcoded value from deserializer" + assert ( + result["2"]("some", "value") == "2" + ), "Expected hardcoded value from deserializer" + assert ( + result["3"]("some", "value") == "3" + ), "Expected hardcoded value from deserializer" diff --git a/tests/workflows/unit_tests/execution_engine/introspection/test_schema_parser.py b/tests/workflows/unit_tests/execution_engine/introspection/test_schema_parser.py index 72cf75818..6d9189581 100644 --- a/tests/workflows/unit_tests/execution_engine/introspection/test_schema_parser.py +++ b/tests/workflows/unit_tests/execution_engine/introspection/test_schema_parser.py @@ -282,7 +282,9 @@ def describe_outputs(cls) -> List[OutputDefinition]: property_description="not available", allowed_references=[ ReferenceDefinition( - selected_element="workflow_image", kind=[IMAGE_KIND] + selected_element="workflow_image", + kind=[IMAGE_KIND], + points_to_batch={True}, ) ], is_list_element=False, @@ -297,6 +299,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: ReferenceDefinition( selected_element="workflow_parameter", kind=[BOOLEAN_KIND, STRING_KIND], + points_to_batch={False}, ) ], is_list_element=False, @@ -309,7 +312,9 @@ def describe_outputs(cls) -> List[OutputDefinition]: property_description="not available", allowed_references=[ ReferenceDefinition( - selected_element="step_output", kind=[IMAGE_KIND] + selected_element="step_output", + kind=[IMAGE_KIND], + points_to_batch={True}, ) ], is_list_element=False, @@ -327,6 +332,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: BOOLEAN_KIND, OBJECT_DETECTION_PREDICTION_KIND, ], + points_to_batch={True}, ) ], is_list_element=False, @@ -338,7 +344,11 @@ def describe_outputs(cls) -> List[OutputDefinition]: property_name="step", property_description="not available", allowed_references=[ - ReferenceDefinition(selected_element="step", kind=[]) + ReferenceDefinition( + selected_element="step", + kind=[], + points_to_batch={False}, + ) ], is_list_element=False, is_dict_element=False, @@ -385,10 +395,14 @@ def describe_outputs(cls) -> List[OutputDefinition]: property_description="not available", allowed_references=[ ReferenceDefinition( - selected_element="workflow_image", kind=[IMAGE_KIND] + selected_element="workflow_image", + kind=[IMAGE_KIND], + points_to_batch={True}, ), ReferenceDefinition( - selected_element="step_output", kind=[IMAGE_KIND] + selected_element="step_output", + kind=[IMAGE_KIND], + points_to_batch={True}, ), # nested list is ignored ], @@ -440,10 +454,14 @@ def describe_outputs(cls) -> List[OutputDefinition]: property_description="not available", allowed_references=[ ReferenceDefinition( - selected_element="workflow_image", kind=[IMAGE_KIND] + selected_element="workflow_image", + kind=[IMAGE_KIND], + points_to_batch={True}, ), ReferenceDefinition( - selected_element="step_output", kind=[IMAGE_KIND] + selected_element="step_output", + kind=[IMAGE_KIND], + points_to_batch={True}, ), # nested list is ignored ], @@ -495,10 +513,14 @@ def describe_outputs(cls) -> List[OutputDefinition]: property_description="not available", allowed_references=[ ReferenceDefinition( - selected_element="workflow_image", kind=[IMAGE_KIND] + selected_element="workflow_image", + kind=[IMAGE_KIND], + points_to_batch={True}, ), ReferenceDefinition( - selected_element="step_output", kind=[IMAGE_KIND] + selected_element="step_output", + kind=[IMAGE_KIND], + points_to_batch={True}, ), # nested list is ignored ], diff --git a/tests/workflows/unit_tests/execution_engine/introspection/test_selectors_parser.py b/tests/workflows/unit_tests/execution_engine/introspection/test_selectors_parser.py index 595cf8d96..fe020eb78 100644 --- a/tests/workflows/unit_tests/execution_engine/introspection/test_selectors_parser.py +++ b/tests/workflows/unit_tests/execution_engine/introspection/test_selectors_parser.py @@ -79,7 +79,9 @@ def describe_outputs(cls) -> List[OutputDefinition]: property_description="not available", allowed_references=[ ReferenceDefinition( - selected_element="workflow_image", kind=[IMAGE_KIND] + selected_element="workflow_image", + kind=[IMAGE_KIND], + points_to_batch={True}, ) ], is_list_element=False, @@ -100,6 +102,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: ReferenceDefinition( selected_element="workflow_parameter", kind=[BOOLEAN_KIND, STRING_KIND], + points_to_batch={False}, ) ], is_list_element=False,