Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enable OS native codecs by default for all pipelines #855

Merged
merged 15 commits into from
Dec 20, 2024
Merged

enable OS native codecs by default for all pipelines #855

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
ba1b94e
trying to enable OS native codec by default
wenbingl Dec 9, 2024
05d08a3
Merge branch 'main' into oscodec
wenbingl Dec 10, 2024
8324814
one more try for iOS build
wenbingl Dec 11, 2024
dbc0a50
adjust MSE threshold
wenbingl Dec 11, 2024
43eb3cd
fix test issue on MacOS
wenbingl Dec 12, 2024
50e609e
Merge branch 'main' into oscodec
wenbingl Dec 12, 2024
28430ec
undo the CoreService change
wenbingl Dec 12, 2024
8f7004b
more corrections
wenbingl Dec 12, 2024
ff16c13
remove iOS support
wenbingl Dec 12, 2024
69e438a
Revert "remove iOS support"
skyline75489 Dec 13, 2024
d73c884
recover
skyline75489 Dec 13, 2024
e48368f
podspec
skyline75489 Dec 13, 2024
61cf0a3
more fixes
skyline75489 Dec 13, 2024
543f135
Merge branch 'main' into oscodec
wenbingl Dec 13, 2024
94ca81b
Merge branch 'main' into oscodec
wenbingl Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
trying to enable OS native codec by default
  • Loading branch information
@wenbingl
wenbingl committed Dec 9, 2024
commit ba1b94ef6bdd12776897ae22e90194e610ab0671
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ option(OCOS_ENABLE_BERT_TOKENIZER "Enable the BertTokenizer building" ON)
option(OCOS_ENABLE_BLINGFIRE "Enable operators depending on the Blingfire library" ON)
option(OCOS_ENABLE_MATH "Enable math tensor operators building" ON)
option(OCOS_ENABLE_DLIB "Enable operators like Inverse depending on DLIB" ON)
option(OCOS_ENABLE_VENDOR_IMAGE_CODECS "Enable and use vendor image codecs if supported over libpng & libjpeg" OFF)
option(OCOS_ENABLE_VENDOR_IMAGE_CODECS "Enable and use vendor image codecs if supported over libpng & libjpeg" ON)
option(OCOS_ENABLE_OPENCV_CODECS "Enable cv2 and vision operators that require opencv imgcodecs." OFF)
option(OCOS_ENABLE_CV2 "Enable the operators in `operators/cv2`" OFF)
option(OCOS_ENABLE_VISION "Enable the operators in `operators/vision`" ON)
Expand Down
65 changes: 41 additions & 24 deletions test/test_tools_add_pre_post_processing_to_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,29 @@
from onnxruntime_extensions.tools import add_pre_post_processing_to_model as add_ppp
from onnxruntime_extensions.tools import add_HuggingFace_CLIPImageProcessor_to_model as add_clip_feature
from onnxruntime_extensions.tools import pre_post_processing as pre_post_processing
from onnxruntime_extensions.tools.pre_post_processing import *
from onnxruntime_extensions.tools.pre_post_processing import * # noqa


script_dir = os.path.dirname(os.path.realpath(__file__))
ort_ext_root = os.path.abspath(os.path.join(script_dir, ".."))
test_data_dir = os.path.join(ort_ext_root, "test", "data", "ppp_vision")


def compare_two_images(image1, image2):
# decoding it firstly to avoid any format issues
image1 = Image.open(io.BytesIO(image1))
image2 = Image.open(io.BytesIO(image2))
if image1.size != image2.size:
return False
# check if the images are similar by MSE
return np.mean(np.square(np.array(image1) - np.array(image2))) < 0.001


def load_image_file(file_path):
with open(file_path, "rb") as f:
return f.read()


# Function to read the mobilenet labels and adjust for PT vs TF training if needed
# def _get_labels(is_pytorch: bool = True):
# labels_file = os.path.join(test_data_dir, "TF.ImageNetLabels.txt")
Expand Down Expand Up @@ -346,10 +361,10 @@ def test_hfbert_tokenizer(self):
self.assertEqual(np.allclose(result[0], ref_output[0]), True)
self.assertEqual(np.allclose(result[1], ref_output[2]), True)
self.assertEqual(np.allclose(result[2], ref_output[1]), True)

def test_hfbert_tokenizer_optional_output(self):
output_model = (self.temp4onnx / "hfbert_tokenizer_optional_output.onnx").resolve()

ref_output = ([
np.array([[2, 236, 118, 16, 1566, 875, 643, 3, 236, 118, 978, 1566, 875, 643, 3]]),
np.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]),
Expand All @@ -368,7 +383,7 @@ def test_hfbert_tokenizer_optional_output(self):
s = ort.InferenceSession(str(output_model), so, providers=["CPUExecutionProvider"])

result = s.run(None, {s.get_inputs()[0].name: np.array([[input_text[0], input_text[1]]])})

self.assertEqual(len(result), 2)

self.assertEqual(np.allclose(result[0], ref_output[0]), True)
Expand Down Expand Up @@ -416,7 +431,7 @@ def draw_boxes_on_image(self, output_model, test_boxes):
so = ort.SessionOptions()
so.register_custom_ops_library(get_library_path())
ort_sess = ort.InferenceSession(str(output_model), providers=['CPUExecutionProvider'], sess_options=so)
image = np.frombuffer(open(Path(test_data_dir)/'wolves.jpg', 'rb').read(), dtype=np.uint8)
image = np.frombuffer(load_image_file(Path(test_data_dir)/'wolves.jpg'), dtype=np.uint8)

return ort_sess.run(None, {'image': image, "boxes_in": test_boxes})[0]

Expand All @@ -432,9 +447,9 @@ def test_draw_box_crop_pad(self):
for idx, is_crop in enumerate([True, False]):
output_img = (Path(test_data_dir) / f"../{ref_img[idx]}").resolve()
create_boxdrawing_model.create_model(output_model, is_crop=is_crop)
image_ref = np.frombuffer(open(output_img, 'rb').read(), dtype=np.uint8)
image_ref = np.frombuffer(load_image_file(output_img), dtype=np.uint8)
output = self.draw_boxes_on_image(output_model, test_boxes[idx])
self.assertEqual((image_ref == output).all(), True)
self.assertTrue(compare_two_images(image_ref, output))

def test_draw_box_share_border(self):
import sys
Expand All @@ -453,8 +468,8 @@ def test_draw_box_share_border(self):
output = self.draw_boxes_on_image(output_model, test_boxes)

output_img = (Path(test_data_dir) / f"../wolves_with_box_share_borders.jpg").resolve()
image_ref = np.frombuffer(open(output_img, 'rb').read(), dtype=np.uint8)
self.assertEqual((image_ref == output).all(), True)
image_ref = np.frombuffer(load_image_file(output_img), dtype=np.uint8)
self.assertTrue(compare_two_images(image_ref, output))

def test_draw_box_off_boundary_box(self):
import sys
Expand All @@ -473,8 +488,8 @@ def test_draw_box_off_boundary_box(self):
output = self.draw_boxes_on_image(output_model, test_boxes)

output_img = (Path(test_data_dir) / f"../wolves_with_box_off_boundary_box.jpg").resolve()
image_ref = np.frombuffer(open(output_img, 'rb').read(), dtype=np.uint8)
self.assertEqual((image_ref == output).all(), True)
image_ref = np.frombuffer(load_image_file(output_img), dtype=np.uint8)
self.assertTrue(compare_two_images(image_ref, output))

def test_draw_box_more_box_by_class_than_colors(self):
import sys
Expand Down Expand Up @@ -503,8 +518,8 @@ def test_draw_box_more_box_by_class_than_colors(self):
output = self.draw_boxes_on_image(output_model, test_boxes)

output_img = (Path(test_data_dir) / f"../wolves_with_box_more_box_than_colors.jpg").resolve()
image_ref = np.frombuffer(open(output_img, 'rb').read(), dtype=np.uint8)
self.assertEqual((image_ref == output).all(), True)
image_ref = np.frombuffer(load_image_file(output_img), dtype=np.uint8)
self.assertTrue(compare_two_images(image_ref, output))

def test_draw_box_more_box_by_score_than_colors(self):
import sys
Expand Down Expand Up @@ -534,8 +549,8 @@ def test_draw_box_more_box_by_score_than_colors(self):
output = self.draw_boxes_on_image(output_model, test_boxes)

output_img = (Path(test_data_dir) / f"../wolves_with_box_more_box_than_colors_score.jpg").resolve()
image_ref = np.frombuffer(open(output_img, 'rb').read(), dtype=np.uint8)
self.assertEqual((image_ref == output).all(), True)
image_ref = np.frombuffer(load_image_file(output_img), dtype=np.uint8)
self.assertTrue(compare_two_images(image_ref, output))

# a box with higher score should be drawn over a box with lower score

Expand All @@ -556,8 +571,8 @@ def test_draw_box_overlapping_with_priority(self):
output = self.draw_boxes_on_image(output_model, test_boxes)

output_img = (Path(test_data_dir) / f"../wolves_with_box_overlapping.jpg").resolve()
image_ref = np.frombuffer(open(output_img, 'rb').read(), dtype=np.uint8)
self.assertEqual((image_ref == output).all(), True)
image_ref = np.frombuffer(load_image_file(output_img), dtype=np.uint8)
self.assertTrue(compare_two_images(image_ref, output))

def test_draw_box_with_large_thickness(self):
import sys
Expand All @@ -576,8 +591,8 @@ def test_draw_box_with_large_thickness(self):
output = self.draw_boxes_on_image(output_model, test_boxes)

output_img = (Path(test_data_dir) / f"../wolves_with_solid_box.jpg").resolve()
image_ref = np.frombuffer(open(output_img, 'rb').read(), dtype=np.uint8)
self.assertEqual((image_ref == output).all(), True)
image_ref = np.frombuffer(load_image_file(output_img), dtype=np.uint8)
self.assertTrue(compare_two_images(image_ref, output))

def _create_pipeline_and_run_for_nms(self, output_model: Path,
has_conf_value: bool,
Expand Down Expand Up @@ -683,7 +698,7 @@ def get_model_output():

out = ort_sess.run(None, {'_input': input_data})[0]
return out

expected_size = [24,12,6,18,12,6,18,12,6,]
idx = 0
for iou_threshold in [0.9, 0.75, 0.5]:
Expand Down Expand Up @@ -996,12 +1011,14 @@ def test_FastestDet(self):
so = ort.SessionOptions()
so.register_custom_ops_library(get_library_path())
ort_sess = ort.InferenceSession(str(output_model), providers=['CPUExecutionProvider'], sess_options=so)
image = np.frombuffer(open(Path(test_data_dir)/'wolves.jpg', 'rb').read(), dtype=np.uint8)
image = np.frombuffer(load_image_file(input_image_path), dtype=np.uint8)

output = ort_sess.run(None, {'image': image})[0]
output_img = (Path(test_data_dir) / f"../wolves_with_fastestDet.jpg").resolve()
image_ref = np.frombuffer(open(output_img, 'rb').read(), dtype=np.uint8)
self.assertEqual((image_ref == output).all(), True)
output_img = (Path(test_data_dir) / "../wolves_with_fastestDet.jpg").resolve()
# output.tofile(str(output_img) + "actual.jpg")

image_ref = np.frombuffer(load_image_file(output_img), dtype=np.uint8)
self.assertTrue(compare_two_images(image_ref, output))


if __name__ == "__main__":
Expand Down
Loading