Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 20 additions & 43 deletions .github/workflows/ci-lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,55 +13,32 @@ name: CI • Lint & Licenses
- "v*.*.*"
pull_request:
workflow_dispatch:
inputs:
docker_image_deeploy:
description: "Deeploy Image to use"
required: false
default: "ghcr.io/pulp-platform/deeploy:devel"

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
select-env:
uses: ./.github/workflows/_select-env.yml
with:
docker_image_deeploy: ${{ inputs.docker_image_deeploy }}

linting:
needs: select-env
runs-on: ${{ needs.select-env.outputs.runner }}
container:
image: ${{ needs.select-env.outputs.image }}
name: Run pre-commit
runs-on: ubuntu-latest
steps:
- name: Checkout Repo
uses: actions/checkout@v4
with:
submodules: recursive
- name: Build Deeploy
shell: bash
run: |
pip install .
pip install -r requirements-dev.txt
- name: Format Python
shell: bash
run: |
yapf -rpd -e "*/TEST_*/" -e "*/third_party/" -e "install/" -e "toolchain/" .
- name: Format Python Imports
shell: bash
run: |
isort --quiet --sg "**/TEST_*/*" --sg "**/third_party/*" --sg "install/*" --sg "toolchain/*" ./ -c
autoflake --quiet -c -r --remove-all-unused-imports --ignore-init-module-imports --exclude "**/third_party/*,**/install/*,**/toolchain/*" .
- name: Format C
shell: bash
run: |
python scripts/run_clang_format.py -e "*/TEST_*/*" -e "*/third_party/*" -e "*/install/*" -e "*/toolchain/*" -r --clang-format-executable=${LLVM_INSTALL_DIR}/bin/clang-format . scripts
- name: Format YAML
shell: bash
fetch-depth: 0

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.11"

- name: Install dependencies
run: |
yamllint .
- name: Check Licenses
shell: bash
python -m pip install --upgrade pip
pip install pre-commit

- name: Run pre-commit (all files)
run: |
python scripts/reuse_skip_wrapper.py $(find . \( -name '*.py' -o -name '*.c' -o -name '*.h' -o -name '*.html' -o -name '*.rst' -o -name '*.yml' -o -name '*.yaml' \) -not -path "*toolchain*" -not -path "*third_party*" -not -path "*prebuilt*" -not -path "*.git/*" -not -path "*install/*" -type f)
# Show diffs on failure and run across the entire repository to ensure consistency
pre-commit run --show-diff-on-failure --all-files

- name: Show git status (debug)
if: ${{ always() }}
run: git status --porcelain || true
1 change: 1 addition & 0 deletions .github/workflows/package-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0

---
name: Package • Publish to PyPi

on:
Expand Down
2 changes: 2 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ exclude: |
| .*TEST_.*
| .*TestFiles.*
| .*runtime.*
| .*prebuilt/.*
)

repos:
Expand All @@ -28,6 +29,7 @@ repos:
- id: reuse
name: Check SPDX License Headers
entry: scripts/reuse_skip_wrapper.py
additional_dependencies: ["reuse==6.2.0"]
language: python
stages: [pre-commit, pre-merge-commit, pre-push, manual]
types: [text]
Expand Down
2 changes: 1 addition & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
},
"labelTransform": {
"text": "${fileDirname}",
"find" : "${workspaceFolder}/DeeployTest/Tests/",
"find": "${workspaceFolder}/DeeployTest/Tests/",
"replace": ""
},
"valueTransform": {
Expand Down
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,21 @@ This file contains the changelog for the Deeploy project. The changelog is divid


### List of Pull Requests
- Use Pre-Commit in CI [#159](https://github.com/pulp-platform/Deeploy/pull/159)
- Deeploy-GAP9 Platform [#143](https://github.com/pulp-platform/Deeploy/pull/143)
- Update CLI interface Across Project, Fix Tutorial, and Remove Legacy Test [#157](https://github.com/pulp-platform/Deeploy/pull/157)

### Added
- Added GAP9 Platform Support: Deployer, Bindings, Templates, Tiler, DMA (L3Dma/MchanDma), target library, CI workflows

### Changed
- Switch CI to use pre-commit for linting
- Update `pulp-nnx` and `pulp-nn-mixed` submodules to their latest versions
- PULP-NN moved to TargetLibraries third-party folder
- Aligned CLI commands across the project

### Fixed
- Fix missing dependency in pre-commit-config
- Fix test paths in Deeploy 101 tutorial

### Removed
Expand Down
34 changes: 14 additions & 20 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,42 +36,36 @@ Additionally, add the title and link to the pull request in the list of pull req
- Remove the link to the precompiled LLVM 12 in the `deeployRunner` for Snitch and in the CI.
[...]
```

## Style guide

Deeploy mainly consists of code implemented in C, Makefile, and Python. To facilitate efficient collaboration among users and contributors, it is important to maintain a consistent coding style. To achieve this, it is strongly recommend to use autoformatting tools with the provided configuration files. Additionally, the Continuous Integration (CI) system checks the adherence to the style guide for each pushed commit. Currently configuration for C using `clang-format` and for Python using `yapf` and `isort` are provided.

You can format all relevant files by running:
```bash
make format
```

Alternatively, to only lint the files without modifying them, you can run:
```bash
make lint
```

### Pre-commit
Deeploy mainly consists of code implemented in C, Makefile, and Python. To facilitate efficient collaboration among users and contributors, it is important to maintain a consistent coding style. We use [pre-commit](https://pre-commit.com) with autoformatting tools to maintain this consistency. Configuration is provided for C using `clang-format` and for Python using `yapf` and `isort`.

Additionally, we provide the [pre-commit](https://pre-commit.com) configuration file which you can use to install github hooks that execute the formatting commands on your changes.
### Setting up pre-commit

You will need to manually install pre-commit since it's not added as a dependency to the `pyproject.toml`:
Install pre-commit (not included in `pyproject.toml`):
```bash
pip install pre-commit
```

The configuration sets the default stage for all the hooks to `pre-push` so to install the git hooks run:
Install the git hooks configured to run at the `pre-push` stage:
```bash
pre-commit install --hook-type pre-push
```
The hooks will run before each push, making sure the pushed code can pass linting checks and not fail the CI on linting.

If you change your mind and don't want the git hooks:
The hooks will automatically format your code before each push, ensuring it passes linting checks and CI validation.

To uninstall the git hooks:
```bash
pre-commit uninstall
```

_Note:_ This configures only the python formatting git hooks. The c formatting is not supported at the moment.
You can also manually run formatting without pushing:
```bash
pre-commit run --all-files

# Or by running the Makefile target:
make format
```

## Licensing

Expand Down
2 changes: 1 addition & 1 deletion Container/Dockerfile.deeploy
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ ENV SOFTHIER_INSTALL_DIR=/app/install/softhier
ENV MINIMALLOC_INSTALL_DIR=/app/install/minimalloc
ENV MEMPOOL_HOME=/app/install/mempool
ENV BENDER_INSTALL_DIR=/app/install/bender
ENV PATH=/app/install/qemu/bin:/app/install/banshee:/app/install/bender:$PATH
ENV PATH=/app/install/qemu/bin:/app/install/banshee:/app/install/bender:$PATH

WORKDIR /app

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,19 @@
int8_t ${nodeName}_log2Core = log2(NUM_CORES);

int32_t ${nodeName}_seq_length = ${size} / ${lastDimLength};
int32_t ${nodeName}_chunk = (${nodeName}_seq_length >> ${nodeName}_log2Core) +
int32_t ${nodeName}_chunk = (${nodeName}_seq_length >> ${nodeName}_log2Core) +
((${nodeName}_seq_length & (NUM_CORES-1)) != 0);
int32_t ${nodeName}_start = MIN(${nodeName}_chunk * ${nodeName}_core_id, ${nodeName}_seq_length);
int32_t ${nodeName}_end = MIN(${nodeName}_start + ${nodeName}_chunk, ${nodeName}_seq_length);

int32_t ${nodeName}_elem_start = ${nodeName}_start * ${lastDimLength};
int32_t ${nodeName}_elem_end = ${nodeName}_end * ${lastDimLength};
int32_t ${nodeName}_elem_count = ${nodeName}_elem_end - ${nodeName}_elem_start;

const float${grad_in_type.referencedType.typeWidth}_t* ${nodeName}_grad_in_ptr = ${grad_in} + ${nodeName}_elem_start;
const float${data_in_type.referencedType.typeWidth}_t* ${nodeName}_data_in_ptr = ${data_in} + ${nodeName}_elem_start;
float${grad_out_type.referencedType.typeWidth}_t* ${nodeName}_grad_out_ptr = ${grad_out} + ${nodeName}_elem_start;

if (${nodeName}_elem_count > 0) {
LayernormGrad_fp${grad_in_type.referencedType.typeWidth}_fp${grad_out_type.referencedType.typeWidth}(
${nodeName}_grad_in_ptr, // Upstream gradient (dy)
Expand Down
2 changes: 1 addition & 1 deletion Deeploy/Targets/PULPOpen/Templates/SGDTemplate.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
temp_mul[3] = learning_rate * ref_${grad}[i+3];
temp_mul[4] = learning_rate * ref_${grad}[i+4];
temp_mul[5] = learning_rate * ref_${grad}[i+5];

// Unrolled subtraction operations
ref_${weight_updated}[i] = ref_${weight}[i] - temp_mul[0];
ref_${weight_updated}[i+1] = ref_${weight}[i+1] - temp_mul[1];
Expand Down
4 changes: 2 additions & 2 deletions DeeployTest/testUtils/core/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
def generate_network(config: DeeployTestConfig, skip: bool = False) -> None:
"""
Generate network code from ONNX model.

Raises:
RuntimeError: If network generation fails
"""
Expand Down Expand Up @@ -151,7 +151,7 @@ def build_binary(config: DeeployTestConfig) -> None:
def run_simulation(config: DeeployTestConfig, skip: bool = False) -> TestResult:
"""
Run simulation and parse output.

Raises:
RuntimeError: If simulation cannot be executed
"""
Expand Down
4 changes: 2 additions & 2 deletions DeeployTest/testUtils/core/paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@
def get_test_paths(test_dir: str, platform: str, base_dir: Optional[str] = None) -> Tuple[str, str, str]:
"""
Resolve test paths for generation and build directories.

Args:
test_dir: Path to test directory (e.g., "Tests/Adder" or absolute path)
platform: Platform name (e.g., "Generic")
base_dir: Base directory for tests (defaults to DeeployTest/)

Returns:
Tuple of (gen_dir, test_dir_abs, test_name)
"""
Expand Down
2 changes: 1 addition & 1 deletion DeeployTest/testUtils/deeployRunner.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ def main(default_platform: Optional[str] = None,
parser_setup_callback = None):
"""
Main entry point for Deeploy test runners.

Args:
default_platform: Default platform if not specified via -p
default_simulator: Default simulator if not specified via -s
Expand Down
4 changes: 2 additions & 2 deletions DeeployTest/testUtils/pytestRunner.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
def get_worker_id() -> str:
"""
Get the pytest-xdist worker ID for parallel test execution.

Returns:
Worker ID string (e.g., 'gw0', 'gw1', 'master' for non-parallel)
"""
Expand Down Expand Up @@ -114,7 +114,7 @@ def create_test_config(
def run_and_assert_test(test_name: str, config: DeeployTestConfig, skipgen: bool, skipsim: bool) -> None:
"""
Shared helper function to run a test and assert its results.

Raises:
AssertionError: If test fails or has errors
"""
Expand Down
6 changes: 3 additions & 3 deletions DeeployTest/test_dmas.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
@pytest.fixture(autouse = True)
def clear_deeploy_state():
"""Clear dynamically generated struct classes from AbstractDataTypes before each test.

This prevents state pollution between DMA tests where dynamically generated
struct classes (like _memcpy_0_tiling_closure_args_t) persist and cause
conflicts when tests with different configurations try to create new versions.
Expand Down Expand Up @@ -111,7 +111,7 @@ def setup_dma_deployer(dma_type: str, input_shape: tuple, tile_shape: tuple, nod
doublebuffer: bool, gen_dir: str):
"""
Set up deployer for DMA testing with custom tiling.

Args:
dma_type: DMA implementation ("MchanDma", "L3Dma", "SnitchDma")
input_shape: Tensor shape to copy
Expand All @@ -120,7 +120,7 @@ def setup_dma_deployer(dma_type: str, input_shape: tuple, tile_shape: tuple, nod
data_type: Data type (uint8_t, uint16_t, uint32_t)
doublebuffer: Enable double buffering
gen_dir: Generation directory

Returns:
tuple: (deployer, test_inputs, test_outputs)
"""
Expand Down
4 changes: 2 additions & 2 deletions DeeployTest/test_platforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,11 @@
def generate_test_params(test_dict, config_name):
"""
Generate test parameters from a dictionary of test names to L1 values.

Args:
test_dict: Dictionary mapping test_name -> list of L1 values
config_name: Configuration name for test ID (e.g., "L2-singlebuffer")

Returns:
List of (test_name, l1_value, config_name) tuples
"""
Expand Down
24 changes: 2 additions & 22 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -556,28 +556,8 @@ chimera-sdk: ${CHIMERA_SDK_INSTALL_DIR}
.PHONY: docs clean-docs format

format:
@echo "Formatting all relevant files..."
@echo " - Format Python Files"
@yapf -ipr -e "*/TEST_*/" -e "*/third_party/" -e "install/" -e "toolchain/" .
@echo " - Format Python Imports"
@isort --quiet --sg "**/TEST_*/*" --sg "**/third_party/*" --sg "install/*" --sg "toolchain/*" ./
@autoflake -i -r --remove-all-unused-imports --ignore-init-module-imports --exclude "**/third_party/*,**/install/*,**/toolchain/*" .
@echo " - Format C/C++ Files"
@python scripts/run_clang_format.py -e "*/TEST_*/*" -e "*/third_party/*" -e "*/install/*" -e "*/toolchain/*" --clang-format-executable=${LLVM_INSTALL_DIR}/bin/clang-format -ir ./ scripts

lint:
@echo "Linting all relevant files..."
@echo " - Lint License Headers"
@scripts/reuse_skip_wrapper.py $$(git ls-files '*.py' '*.c' '*.h' '*.html' '*.rst' '*.yml' '*.yaml')
@echo " - Lint Python Files"
@yapf -rpd -e "*/TEST_*/" -e "*/third_party/" -e "install/" -e "toolchain/" .
@echo " - Lint Python Imports"
@isort --quiet --sg "**/TEST_*/*" --sg "**/third_party/*" --sg "install/*" --sg "toolchain/*" ./ -c
@autoflake --quiet -c -r --remove-all-unused-imports --ignore-init-module-imports --exclude "**/third_party/*,**/install/*,**/toolchain/*" .
@echo " - Lint C/C++ Files"
@python scripts/run_clang_format.py -e "*/TEST_*/*" -e "*/third_party/*" -e "*/install/*" -e "*/toolchain/*" -r --clang-format-executable=${LLVM_INSTALL_DIR}/bin/clang-format . scripts
@echo " - Lint YAML files"
@yamllint .
@echo "Formatting code..."
@pre-commit run --all-files

docs:
make -C docs html
Expand Down
2 changes: 1 addition & 1 deletion pytest.ini
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ minversion = 6.0
pythonpath = .

# Default options
addopts =
addopts =
-ra
--strict-markers
--strict-config
Expand Down
Loading