refactored test environment

Author: andreadesim

.gitignore

@@ -1,10 +1,13 @@
+# build folder for C code
 build/
 
 .vscode/
 __pycache__/
 .idea/
 
+# private path of bpatch exectable
 path_bpatch.py
 
-/test/*
-!/test/*.py
+# test files
+/test/testin/*
+/test/testout/*

bpatch.py

@@ -1,3 +1,15 @@
+#!/usr/bin/python3
+
+"""bpatch.py - Bit Patch Firmware Tool"""
+
+__author__ = "Andrea De Simone and Fabrizio Riente"
+__copyright__ = "Copyright 2025, Politecnico di Torino"
+__license__ = "Apache-2.0"
+__version__ = "1.0.1"
+__maintainer__ = "Andrea De Simone"
+__email__ = "andrea.desimone@polito.it, fabrizio.riente@polito.it"
+
+
 import binascii
 import re
 from math import log, ceil

c/main.h

@@ -6,8 +6,8 @@
 #include "bpatch.h"
 
 // Define the size of the firmware slots in Kilobytes
-#define FW_SLOT_SIZE 2000
+#define FW_SLOT_SIZE 3000
 // Define the size of the patch slot in Kilobytes
-#define PATCH_SLOT_SIZE 1000
+#define PATCH_SLOT_SIZE 2000
 
 #endif

test/README.md

@@ -0,0 +1,32 @@
+# Test Environment
+
+This section describes the environment used to test patch generation and application.  
+It includes two scripts:
+
+- `test.py`: Automatically generates and verifies binary patches.  
+- `plot.py`: Visualizes patch sizes over firmware versions.
+
+## How to Use
+
+### 1. Patch Generation and Verification (`test.py`)
+
+Place one or more folders inside the `testin` directory. Each folder should contain the binaries to be tested.  
+Binaries must include the firmware version in their filename, e.g., `f_v1.bin`, `f_v2.bin`, etc.  
+The script extracts the version number from the last numeric part of the filename.
+
+For each folder in `testin`:
+
+- Patches will be generated and verified.
+- Output will be saved in a corresponding folder in `testout`.
+- Patches will be named `p_n.bin`, where `n` indicates the version number of the new firmware.
+
+### 2. Patch Size Plotting (`plot.py`)
+
+This script generates a plot showing the size of each patch relative to its firmware version.
+
+**Requirements**:
+
+- Patches must be generated using `test.py`.
+- The `matplotlib` Python library is required.
+
+For each folder in `testout`, a separate plot will be generated.

test/plot.py

@@ -1,87 +1,44 @@
 import matplotlib.pyplot as plt
-import numpy as np
-from math import ceil
 import os
 
-from test import extract_last_decimal
-
-for d in os.listdir():
-    if d.startswith('f'):
-        r = {}
-
-        fw_size = np.array([])
-        patch_size = np.array([])
-        n_frag_patch = np.array([])
-        n_frag_fw = np.array([])
-        ovh_nbd = np.array([])
-        ovh_nbc = np.array([])
-        ovh_nba = np.array([])
-        ovh_tot = np.array([])
-        n_nba = np.array([])
-
-        if os.path.exists(d + '/testout'):
-            for report in os.listdir(d + '/testout'):
-                if report.startswith('report'):
-                    # extract version
-                    version = extract_last_decimal(report)
-                    # add to dictionary
-                    r[version] = report
-
-        # check if there are report in the folder
-        if r:
-            fw_versions = sorted(r.keys())
-            for v in fw_versions:
-                with open(d + '/testout/' + r[v], 'r') as f:
-                    f.readline()
-                    fields = f.readline().split(',')
-
-                    fw_size         = np.append(fw_size, int(fields[0]))
-                    patch_size      = np.append(patch_size, int(fields[1]))
-                    n_frag_patch    = np.append(n_frag_patch, ceil(int(fields[1])/112))
-                    n_frag_fw       = np.append(n_frag_fw, ceil(int(fields[0])/112))
-                    n_nba           = np.append(n_nba, int(fields[4]))
-                    ovh_nbd         = np.append(ovh_nbd, float(fields[5]))
-                    ovh_nbc         = np.append(ovh_nbc, float(fields[6]))
-                    ovh_nba         = np.append(ovh_nba, float(fields[7]))
-                    ovh_tot         = np.append(ovh_tot, float(fields[8]))
-
-            # fig, axs = plt.subplots(2, 2, figsize=(15, 10))
-            fig, axs = plt.subplots(2, 2)
-            fig.suptitle(f'Statistics folder: {d}')
-
-            axs[0, 0].set_title('Compression')
-            axs[0, 0].plot(fw_versions, patch_size / fw_size * 100)
-            axs[0, 0].set_xlabel('FW version')
-            axs[0, 0].set_ylabel('Patch compression [%]')
-            axs[0, 0].grid()
-
-            axs[0, 1].set_title('New Bytes')
-            axs[0, 1].plot(fw_versions, np.ceil(n_nba / 1024))
-            axs[0, 1].set_xlabel('FW version')
-            axs[0, 1].set_ylabel('kB')
-            axs[0, 1].grid()
-
-            axs[1, 0].set_title('Overhead')
-            axs[1, 0].plot(fw_versions, ovh_nbd, fw_versions, ovh_nbc, fw_versions, ovh_nba, fw_versions, ovh_tot)
-            axs[1, 0].set_xlabel('FW version')
-            axs[1, 0].set_ylabel('Overhead respect new bytes [%]')
-            axs[1, 0].legend(['nbd', 'nbc', 'nba', 'total'])
-            axs[1, 0].grid()
-
-            width = 0.3  # the width of the bars
-            species = np.arange(len(fw_versions))
-
-            p1 = axs[1, 1].bar(species - width / 2, n_frag_fw, width, label='FW')
-            axs[1, 1].bar_label(p1, label_type='edge')
-            p2 = axs[1, 1].bar(species + width / 2, n_frag_patch, width, label='Patch')
-            axs[1, 1].bar_label(p2, label_type='edge')
-
-            axs[1, 1].set_title('Number of fragment comparison')
-            axs[1, 1].set_xticks(species)
-            axs[1, 1].set_xticklabels(fw_versions)
-            axs[1, 1].legend()
-
-            plt.tight_layout()
-            plt.show()
-
-exit()
+from test import extract_last_decimal, TESTOUT_FOLDER, TESTIN_FOLDER
+
+# exit if TESTOUT_FOLDER does not exist
+if not os.path.exists(TESTOUT_FOLDER):
+    exit(0)
+
+# iterate over all directories in TESTOUT_FOLDER
+for d in os.listdir(TESTOUT_FOLDER):
+    # create patch dict with patch name and version
+    p = {}
+
+    for patch in os.listdir(f"{TESTOUT_FOLDER}/{d}"):
+        # extract the version from the firmware name
+        version = extract_last_decimal(patch)
+        # add patch in dict
+        p[version] = os.path.getsize(f"{TESTOUT_FOLDER}/{d}/{patch}") / 1024
+
+    # create firmware dict with fw name and version
+    fw = {}
+
+    for f in os.listdir(f"{TESTIN_FOLDER}/{d}"):
+        # extract the version from the firmware name
+        version = extract_last_decimal(f)
+        # add patch in dict
+        fw[version] = os.path.getsize(f"{TESTIN_FOLDER}/{d}/{f}") / 1024
+
+    # sort the patch and firmware dicts by version
+    versions = sorted(p.keys())
+    p_sizes = [p[v] for v in versions]
+    f_sizes = [fw[v] for v in versions]
+
+    # plot the firmwares and patch sizes
+    plt.figure()
+    plt.title(f"{d}")
+    plt.xlabel('Firmware Version')
+    plt.ylabel('KBytes')
+    plt.grid()
+    plt.plot(versions, p_sizes, 'o-', label='Patch Size')
+    plt.plot(versions, f_sizes, 'o-', label='Firmware Size')
+    plt.legend()
+    plt.show()

test/test.py

@@ -1,50 +1,102 @@
 import os
-import subprocess
 import re
 import sys
 
+
+# path to testin and testout folders
+TESTIN_FOLDER = "testin"
+TESTOUT_FOLDER = "testout"
+
+# set verbosity level
+"""
+with VERBOSE = 0, only errors and final result is printed
+with VERBOSE = 1, also test start and end for each folder is printed
+with VERBOSE = 2, also a message for each patch is printed
+"""
+VERBOSE = 1
+# set option for bpatch
+OPTIONS = "-d"
+
+
 def extract_last_decimal(s):
+    """
+    Extract the last decimal number from a string.
+
+    :param s: string to extract the last decimal number from
+    :type s: str
+    :return: the last decimal number found in the string, or None if no number is found
+    :rtype: int
+    """
+
     # find all numbers in the string
     numbers = re.findall(r'\d+', s)
     # restur last number if exists
     return numbers[-1]if numbers else None
 
 def test_bpatch():
+    """
+    run the bpatch test on all directories in TESTIN_FOLDER.
+
+    :return: error count
+    :rtype: int
+    """
+
     error_count = 0
+    tested_patches = 0
+
+    # check if testin and testout folders exist, otherwise create them
+    if not os.path.exists(TESTIN_FOLDER):
+        os.makedirs(TESTIN_FOLDER)
+    if not os.path.exists(TESTOUT_FOLDER):
+        os.makedirs(TESTOUT_FOLDER)
 
-    for d in os.listdir():
-        if d.startswith('f'):
+    # iterate over all directories in TESTIN_FOLDER
+    for d in os.listdir(TESTIN_FOLDER):
+        if VERBOSE >= 1:
             print(f'### Start test folder {d} ###')
-            # create output directory if not exists
-            if not os.path.exists(d + '/testout'):
-                os.makedirs(d + '/testout')
-            # create firmware dict with fw name and version
-            f = {}
-            for el in os.listdir(d):
-                # check if the file is a binary file
-                if el.endswith('.bin'):
-                    version = extract_last_decimal(el)
-                    # check if the version is not None and not in the dict
-                    if version is None:
-                        print(f'Error: cannot find version in {el}', file=sys.stderr)
-                        exit(-1)
-                    if version in f:
-                        print(f'Error: duplicate version {version} in {el}', file=sys.stderr)
-                        exit(-1)
-                    # add fw in dict
-                    f[version] = el
-
-            for version in sorted(f.keys())[:-1]:
-                print(f'Start test bpatch for {d}/{f[version]} and {d}/{f[str(int(version) + 1)]}')
-                # run bpatch
-                if os.system(f'python3 ../bpatch.py encode {d}/{f[version]} {d}/{f[str(int(version) + 1)]} {d}/testout/patch_{int(version) + 1}.bin -v -R {d}/testout/report_{int(version) + 1}.csv> /dev/null') != 0:
-                    print(f'Error while running bpatch for {d}/{f[version]} and {d}/{f[str(int(version) + 1)]}', file=sys.stderr)
-                    error_count += 1
-                else:
+
+        # create output directory if not exists or clean previous results
+        if not os.path.exists(f"{TESTOUT_FOLDER}/{d}"):
+            os.makedirs(f"{TESTOUT_FOLDER}/{d}")
+        else:
+            for file in os.listdir(f"{TESTOUT_FOLDER}/{d}"):
+                os.remove(os.path.join(f"{TESTOUT_FOLDER}/{d}", file))
+
+        # create firmware dict with fw name and version
+        f = {}
+        for fw in os.listdir(f"{TESTIN_FOLDER}/{d}"):
+            # extract the version from the firmware name
+            version = extract_last_decimal(fw)
+            # check if the version is not None and not in the dict
+            if version is None:
+                print(f'Error: cannot find version in {fw}', file=sys.stderr)
+                exit(-1)
+            if version in f:
+                print(f'Error: duplicate version {version} in {fw}', file=sys.stderr)
+                exit(-1)
+            # add fw in dict
+            f[version] = fw
+
+        # sort the versions
+        versions = sorted(f.keys())
+
+        for i in range(len(versions) - 1):
+            if VERBOSE >= 2:
+                print(f'Start test bpatch for {d}/{f[versions[i]]} and {d}/{f[versions[i+1]]}')
+            # run bpatch
+            if os.system(f'python3 ../bpatch.py encode {TESTIN_FOLDER}/{d}/{f[versions[i]]} {TESTIN_FOLDER}/{d}/{f[versions[i+1]]} {TESTOUT_FOLDER}/{d}/p_{versions[i+1]}.bin {OPTIONS} -v > /dev/null') != 0:
+                print(f'Error while running bpatch for {d}/{f[versions[i]]} and {d}/{f[versions[i+1]]}', file=sys.stderr)
+                error_count += 1
+            else:
+                if VERBOSE >= 2:
                     print('Test OK')
+            tested_patches += 1
 
+        if VERBOSE >= 1:
             print(f'### End test folder {d} ###\n\n')
 
+    print(f"Test finished\nTotal patches tested: {tested_patches}\nErrors found: {error_count}\n")
+
     return error_count