Add Dataset and Component Class

examples/pipelines/simple_pipeline/build_images.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+function usage {
+  echo "Usage: $0 [options]"
+  echo "Options:"
+  echo "  -c, --component <value>  Set the component name. Pass the component folder name to build a certain components or 'all' to build all components in the current directory (required)"
+  echo "  -n, --namespace <value>  Set the namespace (default: ml6team)"
+  echo "  -r, --repo <value>       Set the repo (default: express)"
+  echo "  -t, --tag <value>        Set the tag (default: latest)"
+  echo "  -h, --help               Display this help message"
+}
+
+# Parse the arguments
+while [[ "$#" -gt 0 ]]; do case $1 in
+  -n|--namespace) namespace="$2"; shift;;
+  -r|--repo) repo="$2"; shift;;
+  -t|--tag) tag="$2"; shift;;
+  -c|--component) component="$2"; shift;;
+  -h|--help) usage; exit;;
+  *) echo "Unknown parameter passed: $1"; exit 1;;
+esac; shift; done
+
+# Check for required argument
+if [ -z "${component}" ]; then
+  echo "Error: component parameter is required"
+  usage
+  exit 1
+fi
+
+# Set default values for optional arguments if not passed
+[ -n "${namespace-}" ] || namespace="ml6team"
+[ -n "${repo-}" ] || repo="express"
+[ -n "${tag-}" ] || tag="latest"
+
+# Get the component directory
+component_dir=$(pwd)/"components"
+
+# Loop through all subdirectories
+for dir in $component_dir/*/; do
+  cd "$dir"
+  BASENAME=${dir%/}
+  BASENAME=${BASENAME##*/}
+  # Build all images or one image depending on the passed argument
+  if [[ "$BASENAME" == "${component}" ]] || [[ "${component}" == "all" ]]; then
+    full_image_name=ghcr.io/${namespace}/${BASENAME}:${tag}
+    echo $full_image_name
+    docker build -t "$full_image_name" \
+     --build-arg COMMIT_SHA=$(git rev-parse HEAD) \
+     --build-arg GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) \
+     --build-arg BUILD_TIMESTAMP=$(date '+%F_%H:%M:%S') \
+     --label org.opencontainers.image.source=https://github.com/${namespace}/${repo} \
+     --platform=linux/arm64 \
+     .
+    docker push "$full_image_name"
+  fi
+  cd "$component_dir"
+done

examples/pipelines/simple_pipeline/components/embedding/embedding.yaml

@@ -0,0 +1,23 @@
+name: Embedding
+description: Component that embeds images using CLIP
+image: embedding:latest
+
+input_subsets:
+
+output_subsets:
+  images:
+    fields:
+      data:
+        type: binary
+  captions:
+    fields:
+      data:
+        type: utf8
+
+args:
+  model_id:
+    description: Model id on the Hugging Face hub
+    type: str
+  batch_size:
+    description: Batch size to use when embedding
+    type: int

examples/pipelines/simple_pipeline/components/image_filtering/Dockerfile

@@ -0,0 +1,29 @@
+FROM --platform=linux/amd64 python:3.8-slim
+
+## System dependencies
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install git curl -y
+
+# Downloading gcloud package
+RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz
+
+# Installing the package
+RUN mkdir -p /usr/local/gcloud \
+&& tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \
+&& /usr/local/gcloud/google-cloud-sdk/install.sh
+
+# Adding the package path to local
+ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
+
+# install requirements
+COPY requirements.txt /
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+# Copy over src-files of the component
+COPY src /src
+
+# Set the working directory to the source folder
+WORKDIR /src
+
+ENTRYPOINT ["python", "main.py"]

examples/pipelines/simple_pipeline/components/image_filtering/kubeflow_component.yaml

@@ -0,0 +1,35 @@
+name: image_filtering
+description: A component that filters images
+inputs: 
+    - name: input_manifest_path
+      description: Path to the input manifest
+      type: String
+
+    - name: min_width
+      description: Desired minimum width
+      type: Integer
+
+    - name: min_height
+      description: Desired minimum height
+      type: Integer
+
+    - name: metadata
+      description: Metadata arguments, passed as a json dict string
+      type: String
+
+
+outputs:
+    - name: output_manifest_path
+      description: Path to the output manifest
+
+implementation:
+    container:
+        image: ghcr.io/ml6team/image_filtering:latest
+        command: [
+            python3, main.py,
+            --input_manifest_path,       {inputPath: input_manifest_path},
+            --min_width,                 {inputValue: min_width},
+            --min_height,                {inputValue: min_height},
+            --metadata,                  {inputValue: metadata},
+            --output_manifest_path,      {outputPath: output_manifest_path},
+        ]

examples/pipelines/simple_pipeline/components/image_filtering/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/ml6team/express.git@b1855308ca9251da5ddd8e6b88c34bc1c082a71b#egg=express[datasets]

examples/pipelines/simple_pipeline/components/image_filtering/src/fondant_component.yaml

@@ -0,0 +1,27 @@
+name: Image filtering
+description: Component that filters images based on desired minimum width and height
+image: image_filtering:latest
+
+input_subsets:
+  images:
+    fields:
+      width:
+        type: int32
+      height:
+        type: int32
+
+output_subsets:
+  images:
+    fields:
+      width:
+        type: int32
+      height:
+        type: int32
+
+args:
+  min_width:
+    description: Desired minimum width
+    type: int
+  min_height:
+    description: Desired minimum height
+    type: int

examples/pipelines/simple_pipeline/components/image_filtering/src/main.py

@@ -0,0 +1,39 @@
+"""
+This component filters images of the dataset based on image size (minimum height and width).
+"""
+import logging
+from typing import Dict
+
+from datasets import Dataset
+
+from express.dataset import FondantComponent
+from express.logger import configure_logging
+
+configure_logging()
+logger = logging.getLogger(__name__)
+
+
+class ImageFilterComponent(FondantComponent):
+    """
+    Component that filters images based on height and width.
+    """
+
+    @classmethod
+    def process(cls, dataset: Dataset, args: Dict) -> Dataset:
+        """
+        Args:
+            dataset
+            args: args to pass to the function
+
+        Returns:
+            dataset
+        """
+        logger.info("Filtering dataset...")
+        min_width, min_height = args.min_width, args.min_height
+        filtered_dataset = dataset.filter(lambda example: example["images_width"] > min_width and example["images_height"] > min_height)
+
+        return filtered_dataset
+
+
+if __name__ == "__main__":
+    ImageFilterComponent.run()

examples/pipelines/simple_pipeline/components/load_from_hub/Dockerfile

@@ -0,0 +1,29 @@
+FROM --platform=linux/amd64 python:3.8-slim
+
+## System dependencies
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install git curl -y
+
+# Downloading gcloud package
+RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz
+
+# Installing the package
+RUN mkdir -p /usr/local/gcloud \
+&& tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \
+&& /usr/local/gcloud/google-cloud-sdk/install.sh
+
+# Adding the package path to local
+ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
+
+# install requirements
+COPY requirements.txt /
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+# Copy over src-files of the component
+COPY src /src
+
+# Set the working directory to the source folder
+WORKDIR /src
+
+ENTRYPOINT ["python", "main.py"]

examples/pipelines/simple_pipeline/components/load_from_hub/kubeflow_component.yaml

@@ -0,0 +1,30 @@
+name: load_from_hub
+description: A component that takes a dataset name from the 🤗 hub as input and uploads it to a GCS bucket.
+inputs: 
+    - name: dataset_name
+      description: Name of dataset on the hub
+      type: String
+
+    - name: batch_size
+      description: Batch size to use to create image metadata
+      type: Integer
+
+    - name: metadata
+      description: Metadata arguments, passed as a json dict string
+      type: String
+
+
+outputs:
+    - name: output_manifest_path
+      description: Path to the output manifest
+
+implementation:
+    container:
+        image: ghcr.io/ml6team/load_from_hub:latest
+        command: [
+            python3, main.py,
+            --dataset_name,              {inputValue: dataset_name},
+            --batch_size,                {inputValue: batch_size},
+            --metadata,                  {inputValue: metadata},
+            --output_manifest_path,      {outputPath: output_manifest_path},
+        ]

examples/pipelines/simple_pipeline/components/load_from_hub/requirements.txt

@@ -0,0 +1,2 @@
+git+https://github.com/ml6team/express.git@b1855308ca9251da5ddd8e6b88c34bc1c082a71b#egg=express[datasets]
+Pillow==9.4.0

examples/pipelines/simple_pipeline/components/load_from_hub/src/fondant_component.yaml

@@ -0,0 +1,31 @@
+name: Load from hub
+description: Component that loads a dataset from the hub
+image: load_from_hub:latest
+
+input_subsets:
+  images:
+      fields:
+        data:
+          type: binary
+
+output_subsets:
+  images:
+    fields:
+      data:
+        type: binary
+      width:
+        type: int32
+      height:
+        type: int32
+  captions:
+    fields:
+      data:
+        type: utf8
+
+args:
+  dataset_name:
+    description: Name of dataset on the hub
+    type: str
+  batch_size:
+    description: Batch size to use to create image metadata
+    type: int