diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index aa3d8e67a..cebb33da0 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -28,3 +28,16 @@ } {% endblock %} + +{% block menu %} +

+ Getting Started +

+ + + +{{super()}} +{% endblock %} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 5e693445a..63189289c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,6 +62,9 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] html_theme = "sphinx_rtd_theme" +html_theme_options = { + 'collapse_navigation': True +} # html_theme = 'alabaster' diff --git a/docs/img/clock.gif b/docs/img/clock.gif new file mode 100644 index 000000000..1597f47e4 Binary files /dev/null and b/docs/img/clock.gif differ diff --git a/docs/img/dash3d_viz.jpg b/docs/img/dash3d_viz.jpg new file mode 100644 index 000000000..d4ad098f8 Binary files /dev/null and b/docs/img/dash3d_viz.jpg differ diff --git a/docs/img/koala.jpg b/docs/img/koala.jpg new file mode 100644 index 000000000..6ea1ea190 Binary files /dev/null and b/docs/img/koala.jpg differ diff --git a/docs/img/ov_viz.jpg b/docs/img/ov_viz.jpg new file mode 100644 index 000000000..2c6836d34 Binary files /dev/null and b/docs/img/ov_viz.jpg differ diff --git a/docs/index.rst b/docs/index.rst index d62ccedb0..c6512bd55 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,12 +7,14 @@ NVIDIA Kaolin library provides a PyTorch API for working with a variety of 3D re Kaolin library is part of a larger suite of tools for 3D deep learning research. For example, `Omniverse Kaolin app `_ will allow interactive visualization of 3D checkpoints. To find out more about the Kaolin ecosystem, visit the `NVIDIA Kaolin Dev Zone page `_. + .. toctree:: :titlesonly: - :caption: Getting Started: + :maxdepth: 1 + :caption: Tutorials: - notes/installation - notes/overview + notes/diff_render + notes/checkpoints .. toctree:: :titlesonly: diff --git a/docs/notes/checkpoints.rst b/docs/notes/checkpoints.rst new file mode 100644 index 000000000..4dd361166 --- /dev/null +++ b/docs/notes/checkpoints.rst @@ -0,0 +1,146 @@ +3D Checkpoints +============== + +.. image:: ../img/koala.jpg + +Visualizing 3D inputs and outputs of your model during training is an +essential diagnostic tool. Kaolin provides a :ref:`simple API to checkpoint` **batches of meshes, pointclouds and voxelgrids**, as well as **colors and +textures**, saving them in :ref:`the USD format`. These checkpoints can then be visualized locally using :ref:`Kaolin Omniverse App` or by launching :ref:`Kaolin Dash3D` on the commandline, allowing remote visualization through a web browser. + +.. _writing checkpoints: + +Writing Checkpoints: +-------------------- + +In a common scenario, model performance is visualized for a +small evaluation batch. Bootstrap 3D checkpoints in your python training +code by configuring a :class:`~kaolin.visualize.Timelapse` object:: + + import kaolin + timelapse = kaolin.visualize.Timelapse(viz_log_dir) + +The ``viz_log_dir`` is the directory where checkpoints will be saved. Timelapse will create files and subdirectories under this path, so providing +a dedicated ``viz_log_dir`` separate from your other logs and configs will help keep things clean. The :class:`~kaolin.visualize.Timelapse` API supports point clouds, +voxel grids and meshes, as well as colors and textures. + +Saving Fixed Data +^^^^^^^^^^^^^^^^^ + +To save any iteration-independent data, +call ``timelapse`` before your training loop +without providing an ``iteration`` parameter, e.g.:: + + timelapse.add_mesh_batch(category='ground_truth', + faces_list=face_list, + vertices_list=gt_vert_list) + timelapse.add_pointcloud_batch(category='input', + pointcloud_list=input_pt_clouds) +The ``category`` identifies the meaning of the data. In this toy example, +the model learns to turn the ``'input'`` pointcloud into the ``'output'`` mesh. Both the ``'ground_truth'`` mesh and the ``'input'`` pointcloud batches are only saved once for easy visual comparison. + +Saving Time-varying Data +^^^^^^^^^^^^^^^^^^^^^^^^ +To checkpoint time-varying data during training, simply call :meth:`~kaolin.visualize.Timelapse.add_mesh_batch`, :meth:`~kaolin.visualize.Timelapse.add_pointcloud_batch` or :meth:`~kaolin.visualize.Timelapse.add_voxelgrid_batch`, for example:: + + if iteration % checkpoint_interval == 0: + timelapse.add_mesh_batch(category='output', + iteration=iteration, + faces_list=face_list, + vertices_list=out_vert_list) + +.. Tip:: + For any data type, only time-varying data needs to be saved at every iteration. E.g., if your output mesh topology is fixed, only save ``faces_list`` once, and then call ``add_mesh_batch`` with only the predicted ``vertices_list``. This will cut down your checkpoint size. + +Saving Colors and Appearance +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We are working on adding support for colors and semantic ids to +point cloud and voxel grid checkpoints. Mesh API supports multiple time-varying materials +by specifying a :class:`kaolin.io.PBRMaterial`. For an example +of using materials, see +`test_timelapse.py `_. + +Sample Code +^^^^^^^^^^^ +We provide a `script `_ that writes mock checkpoints, which can be run as follows:: + + python examples/tutorial/visualize_main.py \ + --test_objs=path/to/object1.obj,path/to/object2.obj \ + --output_dir=path/to/logdir + +In addition, see :ref:`diff_render` tutorial. + +.. _file format: + +Understanding the File Format: +------------------------------ + +Kaolin :class:`~kaolin.visualize.Timelapse` writes checkpoints using Universal Scene Descriptor (USD) file format (`Documentation `_), developed with wide support for use cases in visual effects, including time-varying data. This allows reducing redundancy in written +data across time. + +After checkpointing with :class:`~kaolin.visualize.Timelapse`, the input ``viz_log_dir`` will contain +a similar file structure:: + + ground_truth/mesh_0.usd + ground_truth/mesh_1.usd + ground_truth/mesh_... + ground_truth/textures + input/pointcloud_0.usd + input/pointcloud_1.usd + input/pointcloud_... + output/mesh_0.usd + output/mesh_1.usd + output/mesh_... + output/pointcloud_0.usd + output/pointcloud_1.usd + output/pointcloud_... + output/textures + +Here, the root folder names correspond to the ``category`` parameter +provided to :class:`~kaolin.visualize.Timelapse` functions. Each element +of the batch of every type is saved in its own numbered ``.usd`` file. Each USD file can be viewed on its +own using any USD viewer, such as `NVIDIA Omniverse View `_, or the whole log directory can be visualized +using the tools below. + +.. Caution:: + Timelapse is designed to only save one visualization batch for every category and type. Saving multiple batches without interleaving the data can be accomplished by creating custom categories. + +.. _ov app: + +Visualizing with Kaolin Omniverse App: +-------------------------------------- + +.. image:: ../img/ov_viz.jpg + +USD checkpoints can be visualized using a dedicated Omniverse Kaolin App `Training Visualizer `_. +This extension provides full-featured support and high-fidelity rendering +of all data types and materials that can be exported using :class:`~kaolin.visualize.Timelapse`, and allows creating custom visualization layouts and viewing meshes in multiple time-varying materials. `Download NVIDIA Omniverse `_ to get started! + +.. _dash 3d: + +Visualizing with Kaolin Dash3D: +------------------------------- + +.. image:: ../img/dash3d_viz.jpg + +Omniverse app requires local access to a GPU and to the saved checkpoints, which is not always possible. +We are also developing a lightweight ``kaolin-dash3d`` visualizer, +which allows visualizing local and remote checkpoints without specialized +hardware or applications. This tool is bundled with the latest +builds as a command-line utility (**Important**: set ``KAOLIN_INSTALL_EXPERIMENTAL=1`` to install; see :ref:`installation`). + +To start Dash3D on the machine that stores the checkpoints, run:: + + kaolin-dash3d --logdir=$TIMELAPSE_DIR --port=8080 + +The ``logdir`` is the directory :class:`kaolin.visualize.Timelapse` was configured with. This command will launch a web server that will stream +geometry to web clients. To connect, simply visit ``http://ip.of.machine:8080`` (or `localhost:8080 `_ if connecting locally or with ssh port forwarding). + +Try it now: +^^^^^^^^^^^^^ + +See Dash3D in action by running it on our test samples and visiting `localhost:8080 `_:: + + kaolin-dash3d --logdir=$KAOLIN_ROOT/tests/samples/timelapse/notexture/ --port=8080 + +.. Caution:: Dash3d is still an experimental feature under active development. It only supports **triangle meshes** and **pointclouds** and cannot yet visualize colors, ids or textures. The web client was tested the most on `Google Chrome `_. We welcome your early feedback on our `github `_! \ No newline at end of file diff --git a/docs/notes/diff_render.rst b/docs/notes/diff_render.rst new file mode 100644 index 000000000..362ccd879 --- /dev/null +++ b/docs/notes/diff_render.rst @@ -0,0 +1,13 @@ +.. _diff_render: + +Differentiable Rendering +======================== + +.. image:: ../img/clock.gif + +Differentiable rendering can be used to optimize the underlying 3D properties, like geometry and lighting, by backpropagating gradients from the loss in the image space. We provide an end-to-end tutorial for using the :mod:`kaolin.render.mesh` API in a Jupyter notebook: + +`examples/tutorial/dibr_tutorial.ipynb `_ + +In addition to the rendering API, the tutorial uses Omniverse Kaolin App `Data Generator `_ to create training data, :class:`kaolin.visualize.Timelapse` to write checkpoints, and +Omniverse Kaolin App `Training Visualizer `_ to visualize them. \ No newline at end of file diff --git a/docs/notes/installation.rst b/docs/notes/installation.rst index ea7c2c121..51656da31 100644 --- a/docs/notes/installation.rst +++ b/docs/notes/installation.rst @@ -1,3 +1,5 @@ +.. _installation: + Installation ============ diff --git a/docs/notes/overview.rst b/docs/notes/overview.rst index 07c5e8c4f..472685102 100644 --- a/docs/notes/overview.rst +++ b/docs/notes/overview.rst @@ -1,5 +1,7 @@ -Overview -======== +.. _overview: + +API Overview and Reference +========================== Kaolin library --------------