From 3e886d8f0ac55c416dae51c1c2661e16eb34718e Mon Sep 17 00:00:00 2001 From: Dylan Ebert Date: Wed, 14 Feb 2024 14:10:52 -0500 Subject: [PATCH] Model3D Gaussian Splatting (#7406) * add gsplat dependency add gsplat implementation run formatting move example .splat and .ply to url add changeset dispose of renderer on unmount notebook * dynamically load canvas components --- .changeset/tough-baboons-greet.md | 6 ++ demo/model3D/run.ipynb | 2 +- demo/model3D/run.py | 2 + gradio/components/model3d.py | 4 +- js/model3D/package.json | 3 +- js/model3D/shared/Canvas3DGS.svelte | 130 +++++++++++++++++++++++++ js/model3D/shared/Model3D.svelte | 58 ++++++++--- js/model3D/shared/Model3DUpload.svelte | 56 ++++++++--- pnpm-lock.yaml | 7 ++ 9 files changed, 241 insertions(+), 27 deletions(-) create mode 100644 .changeset/tough-baboons-greet.md create mode 100644 js/model3D/shared/Canvas3DGS.svelte diff --git a/.changeset/tough-baboons-greet.md b/.changeset/tough-baboons-greet.md new file mode 100644 index 0000000000000..25fbb8f05a791 --- /dev/null +++ b/.changeset/tough-baboons-greet.md @@ -0,0 +1,6 @@ +--- +"@gradio/model3d": minor +"gradio": minor +--- + +feat:Model3D Gaussian Splatting diff --git a/demo/model3D/run.ipynb b/demo/model3D/run.ipynb index e44bd515ac7f5..93d8e7e114822 100644 --- a/demo/model3D/run.ipynb +++ b/demo/model3D/run.ipynb @@ -1 +1 @@ -{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: model3D"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/Bunny.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Bunny.obj\n", "!wget -q -O files/Duck.glb https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Duck.glb\n", "!wget -q -O files/Fox.gltf https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Fox.gltf\n", "!wget -q -O files/face.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/face.obj\n", "!wget -q -O files/sofia.stl https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/sofia.stl\n", "!wget -q -O files/source.txt https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/source.txt"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import os\n", "\n", "\n", "def load_mesh(mesh_file_name):\n", " return mesh_file_name\n", "\n", "\n", "demo = gr.Interface(\n", " fn=load_mesh,\n", " inputs=gr.Model3D(),\n", " outputs=gr.Model3D(\n", " clear_color=[0.0, 0.0, 0.0, 0.0], label=\"3D Model\"),\n", " examples=[\n", " [os.path.join(os.path.abspath(''), \"files/Bunny.obj\")],\n", " [os.path.join(os.path.abspath(''), \"files/Duck.glb\")],\n", " [os.path.join(os.path.abspath(''), \"files/Fox.gltf\")],\n", " [os.path.join(os.path.abspath(''), \"files/face.obj\")],\n", " [os.path.join(os.path.abspath(''), \"files/sofia.stl\")],\n", " ],\n", " cache_examples=True\n", ")\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5} \ No newline at end of file +{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: model3D"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/Bunny.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Bunny.obj\n", "!wget -q -O files/Duck.glb https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Duck.glb\n", "!wget -q -O files/Fox.gltf https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Fox.gltf\n", "!wget -q -O files/face.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/face.obj\n", "!wget -q -O files/sofia.stl https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/sofia.stl\n", "!wget -q -O files/source.txt https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/source.txt"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import os\n", "\n", "\n", "def load_mesh(mesh_file_name):\n", " return mesh_file_name\n", "\n", "\n", "demo = gr.Interface(\n", " fn=load_mesh,\n", " inputs=gr.Model3D(),\n", " outputs=gr.Model3D(\n", " clear_color=[0.0, 0.0, 0.0, 0.0], label=\"3D Model\"),\n", " examples=[\n", " [os.path.join(os.path.abspath(''), \"files/Bunny.obj\")],\n", " [os.path.join(os.path.abspath(''), \"files/Duck.glb\")],\n", " [os.path.join(os.path.abspath(''), \"files/Fox.gltf\")],\n", " [os.path.join(os.path.abspath(''), \"files/face.obj\")],\n", " [os.path.join(os.path.abspath(''), \"files/sofia.stl\")],\n", " [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bonsai/bonsai-7k-mini.splat\"],\n", " [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/luigi/luigi.ply\"],\n", " ],\n", " cache_examples=True\n", ")\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5} \ No newline at end of file diff --git a/demo/model3D/run.py b/demo/model3D/run.py index 232d7aac53f3a..acd3f14652b38 100644 --- a/demo/model3D/run.py +++ b/demo/model3D/run.py @@ -17,6 +17,8 @@ def load_mesh(mesh_file_name): [os.path.join(os.path.dirname(__file__), "files/Fox.gltf")], [os.path.join(os.path.dirname(__file__), "files/face.obj")], [os.path.join(os.path.dirname(__file__), "files/sofia.stl")], + ["https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bonsai/bonsai-7k-mini.splat"], + ["https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/luigi/luigi.ply"], ], cache_examples=True ) diff --git a/gradio/components/model3d.py b/gradio/components/model3d.py index a11d8ae6b1a6b..034d43096327d 100644 --- a/gradio/components/model3d.py +++ b/gradio/components/model3d.py @@ -15,7 +15,7 @@ @document() class Model3D(Component): """ - Creates a component allows users to upload or view 3D Model files (.obj, .glb, .stl, or .gltf). + Creates a component allows users to upload or view 3D Model files (.obj, .glb, .stl, .gltf, .splat, or .ply). Demos: model3D Guides: how-to-use-3D-model-component @@ -54,7 +54,7 @@ def __init__( ): """ Parameters: - value: path to (.obj, .glb, .stl, or .gltf) file to show in model3D viewer. If callable, the function will be called whenever the app loads to set the initial value of the component. + value: path to (.obj, .glb, .stl, .gltf, .splat, or .ply) file to show in model3D viewer. If callable, the function will be called whenever the app loads to set the initial value of the component. clear_color: background color of scene, should be a tuple of 4 floats between 0 and 1 representing RGBA values. camera_position: initial camera position of scene, provided as a tuple of `(alpha, beta, radius)`. Each value is optional. If provided, `alpha` and `beta` should be in degrees reflecting the angular position along the longitudinal and latitudinal axes, respectively. Radius corresponds to the distance from the center of the object to the camera. zoom_speed: the speed of zooming in and out of the scene when the cursor wheel is rotated or when screen is pinched on a mobile device. Should be a positive float, increase this value to make zooming faster, decrease to make it slower. Affects the wheelPrecision property of the camera. diff --git a/js/model3D/package.json b/js/model3D/package.json index a048163547892..c1239ccce24d3 100644 --- a/js/model3D/package.json +++ b/js/model3D/package.json @@ -17,7 +17,8 @@ "@types/babylon": "^6.16.6", "babylonjs": "^4.2.1", "babylonjs-loaders": "^4.2.1", - "dequal": "^2.0.2" + "dequal": "^2.0.2", + "gsplat": "^1.0.5" }, "main_changeset": true, "main": "./Index.svelte", diff --git a/js/model3D/shared/Canvas3DGS.svelte b/js/model3D/shared/Canvas3DGS.svelte new file mode 100644 index 0000000000000..b34c1e98f2fc4 --- /dev/null +++ b/js/model3D/shared/Canvas3DGS.svelte @@ -0,0 +1,130 @@ + + + diff --git a/js/model3D/shared/Model3D.svelte b/js/model3D/shared/Model3D.svelte index c127c11fb9c7b..5fdacd66fad02 100644 --- a/js/model3D/shared/Model3D.svelte +++ b/js/model3D/shared/Model3D.svelte @@ -2,7 +2,6 @@ import type { FileData } from "@gradio/client"; import { BlockLabel, IconButton } from "@gradio/atoms"; import { File, Download, Undo } from "@gradio/icons"; - import Canvas3D from "./Canvas3D.svelte"; import type { I18nFormatter } from "@gradio/utils"; import { dequal } from "dequal"; @@ -22,9 +21,21 @@ let current_settings = { camera_position, zoom_speed, pan_speed }; - let canvas3d: Canvas3D; + let canvas3dgs: any; + let canvas3d: any; + let use_3dgs = false; let resolved_url: string | undefined; + async function loadCanvas3D(): Promise { + const module = await import("./Canvas3D.svelte"); + return module.default; + } + + async function loadCanvas3DGS(): Promise { + const module = await import("./Canvas3DGS.svelte"); + return module.default; + } + function handle_undo(): void { canvas3d.reset_camera_position(camera_position, zoom_speed, pan_speed); } @@ -39,6 +50,21 @@ current_settings = { camera_position, zoom_speed, pan_speed }; } } + + $: { + if (value) { + use_3dgs = value?.path.endsWith(".splat") || value?.path.endsWith(".ply"); + if (use_3dgs) { + loadCanvas3DGS().then((module) => { + canvas3dgs = module; + }); + } else { + loadCanvas3D().then((module) => { + canvas3d = module; + }); + } + } + } - + {#if use_3dgs} + + {:else} + + {/if} {/if} diff --git a/js/model3D/shared/Model3DUpload.svelte b/js/model3D/shared/Model3DUpload.svelte index fd03c46c84883..e2b75726ffce3 100644 --- a/js/model3D/shared/Model3DUpload.svelte +++ b/js/model3D/shared/Model3DUpload.svelte @@ -5,7 +5,6 @@ import { BlockLabel } from "@gradio/atoms"; import { File } from "@gradio/icons"; import type { I18nFormatter } from "@gradio/utils"; - import Canvas3D from "./Canvas3D.svelte"; export let value: null | FileData; export let clear_color: [number, number, number, number] = [0, 0, 0, 0]; @@ -39,9 +38,22 @@ dispatch("change"); } - let canvas3D: Canvas3D; + let canvas3d: any; + let canvas3dgs: any; + let use_3dgs = false; + + async function loadCanvas3D(): Promise { + const module = await import("./Canvas3D.svelte"); + return module.default; + } + + async function loadCanvas3DGS(): Promise { + const module = await import("./Canvas3DGS.svelte"); + return module.default; + } + async function handle_undo(): Promise { - canvas3D.reset_camera_position(camera_position, zoom_speed, pan_speed); + canvas3d.reset_camera_position(camera_position, zoom_speed, pan_speed); } const dispatch = createEventDispatcher<{ @@ -54,6 +66,21 @@ let dragging = false; $: dispatch("drag", dragging); + + $: { + if (value) { + use_3dgs = value?.path.endsWith(".splat") || value?.path.endsWith(".ply"); + if (use_3dgs) { + loadCanvas3DGS().then((module) => { + canvas3dgs = module; + }); + } else { + loadCanvas3D().then((module) => { + canvas3d = module; + }); + } + } + } @@ -62,7 +89,7 @@ @@ -76,14 +103,19 @@ on:undo={handle_undo} absolute /> - + + {#if use_3dgs} + + {:else} + + {/if} {/if} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3ddc4a3a33495..cb312149d6501 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1182,6 +1182,9 @@ importers: dequal: specifier: ^2.0.2 version: 2.0.3 + gsplat: + specifier: ^1.0.5 + version: 1.0.5 js/number: dependencies: @@ -10985,6 +10988,10 @@ packages: engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} dev: false + /gsplat@1.0.5: + resolution: {integrity: sha512-SM85+qMA/UwdDk2lFRmkJvLhD5A+qloRzINmPzjvXJ6Iugl3yAEr8TomVQmge0z1i98bHEoSZrJC/UOKVXq3Hg==} + dev: false + /gunzip-maybe@1.4.2: resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==} hasBin: true