[ESPCN] get_models.py update and addition of extra verification steps

build/espcn/build.py

@@ -26,17 +26,22 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from custom_steps import *
+from custom_steps import (
+    # custom_step_export_verification,
+    custom_step_qonnx_tidy_up,
+    custom_step_add_pre_proc,
+    custom_step_streamline,
+    custom_step_convert_to_hls,
+)
 
 import finn.builder.build_dataflow as build
 import finn.builder.build_dataflow_config as build_cfg
-from finn.builder.build_dataflow_steps import *
-
-model_name = "espcn-bsd300"
 
+model_name = "espcn-bsd300-RCNNTDC"
 
 
 espcn_build_steps = [
+    # custom_step_export_verification,
     custom_step_qonnx_tidy_up,
     custom_step_add_pre_proc,
     "step_qonnx_to_finn",
@@ -65,23 +70,27 @@
     steps=espcn_build_steps,
     output_dir="output_%s_kriasom" % (model_name),
     synth_clk_period_ns=5.0,
-    target_fps=10000,
+    target_fps=30,
     fpga_part="xck26-sfvc784-2LV-c",
-    shell_flow_type = build_cfg.ShellFlowType.VIVADO_ZYNQ,
-    board = "KV260_SOM",
-    enable_build_pdb_debug=False,
-    verbose=False,
-    split_large_fifos = True,
-    folding_config_file = "folding_config_chrc_cap.json",
-    auto_fifo_depths = False,
-    rtlsim_batch_size = 100,
-    verify_input_npy = "quant_espcn_x2_w4a4_base/input.npy",
-    verify_expected_output_npy = "quant_espcn_x2_w4a4_base/output.npy",
+    shell_flow_type=build_cfg.ShellFlowType.VIVADO_ZYNQ,
+    board="KV260_SOM",
+    enable_build_pdb_debug=True,
+    verbose=True,
+    split_large_fifos=True,
+    folding_config_file="folding_config_rcnntdc_cap.json",
+    auto_fifo_depths=False,
+    # auto_fifo_strategy = build_cfg.AutoFIFOSizingMethod.CHARACTERIZE,
+    rtlsim_batch_size=100,
+    force_rtl_conv_inp_gen=False,
+    # start_step="step_hls_ipgen",
+    # stop_step = "step_generate_estimate_reports",
+    verify_input_npy="quant_espcn_x2_w4a4_base/input.npy",
+    verify_expected_output_npy="quant_espcn_x2_w4a4_base/output.npy",
     verify_steps=[
-        build_cfg.VerificationStepType.QONNX_TO_FINN_PYTHON,
-        build_cfg.VerificationStepType.TIDY_UP_PYTHON,
-        build_cfg.VerificationStepType.STREAMLINED_PYTHON,
-        build_cfg.VerificationStepType.FOLDED_HLS_CPPSIM,
+        # build_cfg.VerificationStepType.QONNX_TO_FINN_PYTHON,
+        # build_cfg.VerificationStepType.TIDY_UP_PYTHON,
+        # build_cfg.VerificationStepType.STREAMLINED_PYTHON,
+        # build_cfg.VerificationStepType.FOLDED_HLS_CPPSIM,True
     ],
     generate_outputs=[
         build_cfg.DataflowOutputType.ESTIMATE_REPORTS,

build/espcn/custom_steps.py

@@ -42,26 +42,38 @@
 from qonnx.transformation.infer_shapes import InferShapes
 from qonnx.transformation.merge_onnx_models import MergeONNXModels
 from qonnx.transformation.subpixel_to_deconv import SubPixelToDeconvolution
+from qonnx.transformation.resize_conv_to_deconv import ResizeConvolutionToDeconvolution
 from qonnx.transformation.lower_convs_to_matmul import LowerConvsToMatMul
 from qonnx.transformation.infer_data_layouts import InferDataLayouts
 
 import finn.transformation.streamline.absorb as absorb
 import finn.transformation.fpgadataflow.convert_to_hls_layers as to_hls
 from finn.transformation.streamline import Streamline
 from finn.transformation.streamline.round_thresholds import RoundAndClipThresholds
-from finn.transformation.streamline.reorder import MoveScalarMulPastConvTranspose
-from finn.transformation.move_reshape import RemoveCNVtoFCFlatten
+from finn.transformation.streamline.reorder import (
+    MoveScalarMulPastConvTranspose,
+    MakeScaleResizeNHWC,
+)
 from finn.transformation.fpgadataflow.infer_pixel_padding_deconv import InferPixelPaddingDeconv
 
 from finn.builder.build_dataflow_config import DataflowBuildConfig, VerificationStepType
 from finn.builder.build_dataflow_steps import verify_step
 from finn.util.pytorch import ToTensor
 
+
+def custom_step_export_verification(model: ModelWrapper, cfg: DataflowBuildConfig):
+    model = model.transform(InferShapes())
+    # verify_step(model, cfg, "onnx_export", need_parent=False)
+    return model
+
+
 def custom_step_qonnx_tidy_up(model: ModelWrapper, cfg: DataflowBuildConfig):
     model = model.transform(InferShapes())
     # QONNX transformations
     model = model.transform(SubPixelToDeconvolution())
+    model = model.transform(ResizeConvolutionToDeconvolution(maintain_bit_width=False))
     model = model.transform(InferShapes())
+    # verify_step(model, cfg, "tidy_up", need_parent=False)
     return model
 
 
@@ -86,9 +98,11 @@ def custom_step_add_pre_proc(model: ModelWrapper, cfg: DataflowBuildConfig):
     model = model.transform(InferDataTypes())
     model = model.transform(RemoveStaticGraphInputs())
     model = model.transform(RemoveUnusedTensors())
+    # verify_step(model, cfg, "pre_proc", need_parent=False)
 
     return model
 
+
 def custom_step_streamline(model: ModelWrapper, cfg: DataflowBuildConfig):
     """Run streamlining on given model. Streamlining involves moving floating point
     scale/shift parameters around, collapsing adjacent ones into a single parameter,
@@ -107,8 +121,10 @@ def custom_step_streamline(model: ModelWrapper, cfg: DataflowBuildConfig):
         model = model.transform(LowerConvsToMatMul())
         model = model.transform(absorb.AbsorbConsecutiveTransposes())
         model = model.transform(absorb.AbsorbTransposeIntoMultiThreshold())
-  
+
     model = model.transform(Streamline())
+    model = model.transform(InferDataLayouts())
+    model = model.transform(MakeScaleResizeNHWC())
     model = model.transform(absorb.AbsorbConsecutiveTransposes())
     model = model.transform(InferDataLayouts())
     model = model.transform(RemoveUnusedTensors())
@@ -124,7 +140,6 @@ def custom_step_convert_to_hls(model: ModelWrapper, cfg: DataflowBuildConfig):
     layers. Which nodes and particular configurations can be converted to HLS
     is limited, see the source code of the `convert_to_hls` module for more."""
 
-
     mem_mode = cfg.default_mem_mode.value
     if cfg.standalone_thresholds:
         # doing this first causes all threshold layers to be standalone
@@ -134,6 +149,9 @@ def custom_step_convert_to_hls(model: ModelWrapper, cfg: DataflowBuildConfig):
         model = model.transform(InferPixelPaddingDeconv())
         model = model.transform(absorb.AbsorbTransposeIntoMultiThreshold())
         model = model.transform(RoundAndClipThresholds())
+    need_upsample = len(model.get_nodes_by_op_type("Resize")) > 0
+    if need_upsample:
+        model = model.transform(to_hls.InferUpsample())
     # needed for non-bipolar MatMul layers
     model = model.transform(to_hls.InferQuantizedMatrixVectorActivation(mem_mode))
     # input quantization (if any) as standalone threshold

build/espcn/folding_config_chrc_cap.json

@@ -518,4 +518,4 @@
       0
     ]
   }
-}
+}