From 1da1e7ba4f082539226aa52b0f36d36232107813 Mon Sep 17 00:00:00 2001 From: Tianyi Zheng Date: Fri, 13 Jan 2023 15:30:53 -0500 Subject: [PATCH 01/16] super already calls forward_graph connection --- analogvnn/nn/module/FullSequential.py | 1 - 1 file changed, 1 deletion(-) diff --git a/analogvnn/nn/module/FullSequential.py b/analogvnn/nn/module/FullSequential.py index b99a7b5..657ff1b 100644 --- a/analogvnn/nn/module/FullSequential.py +++ b/analogvnn/nn/module/FullSequential.py @@ -24,6 +24,5 @@ def compile(self, device: Optional[torch.device] = None, layer_data: bool = True FullSequential: self """ arr = [self.graphs.INPUT, *list(self._runtime_module_list.values()), self.graphs.OUTPUT] - self.graphs.forward_graph.add_connection(*arr) self.graphs.backward_graph.add_connection(*reversed(arr)) return super().compile(device, layer_data) From 4b9b32d07c17358a235ac7754a7c2474f6c22c4d Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 17:34:35 -0500 Subject: [PATCH 02/16] added CODEOWNERS --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..e3252aa --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @vivswan \ No newline at end of file From a107fe225204681b28311f6b534a977b2541b35c Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 17:35:36 -0500 Subject: [PATCH 03/16] add not compiled error message --- analogvnn/nn/module/Model.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/analogvnn/nn/module/Model.py b/analogvnn/nn/module/Model.py index bb8ba0d..fb40260 100644 --- a/analogvnn/nn/module/Model.py +++ b/analogvnn/nn/module/Model.py @@ -94,6 +94,24 @@ def use_autograd_graph(self, use_autograd_graph: bool): """ self.graphs.use_autograd_graph = use_autograd_graph + def __call__(self, *args, **kwargs): + """Call the model. + + Args: + *args: The arguments of the model. + **kwargs: The keyword arguments of the model. + + Returns: + TENSORS: The output of the model. + + Raises: + RuntimeError: if the model is not compiled. + """ + if not self._compiled: + raise RuntimeError('Model is not compiled yet.') + + return super(Model, self).__call__(*args, **kwargs) + def compile(self, device: Optional[torch.device] = None, layer_data: bool = True): """Compile the model. From 9a2afa3de52f89bc154f82dae53702463c117aa9 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 17:36:06 -0500 Subject: [PATCH 04/16] added local version exception --- analogvnn/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/analogvnn/__init__.py b/analogvnn/__init__.py index 73955d4..32f5812 100644 --- a/analogvnn/__init__.py +++ b/analogvnn/__init__.py @@ -3,9 +3,13 @@ import importlib_metadata __package__ = 'analogvnn' -__version__ = importlib_metadata.version(__package__) __author__ = 'Vivswan Shah (vivswanshah@pitt.edu)' +try: + __version__ = importlib_metadata.version(__package__) +except importlib_metadata.PackageNotFoundError: + __version__ = 'local' + if sys.version_info < (3, 7, 0): import warnings From ab8183cf0994d858ac69092487206f63c9f5ca85 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 17:36:28 -0500 Subject: [PATCH 05/16] improved docstring --- analogvnn/graph/ModelGraphState.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/analogvnn/graph/ModelGraphState.py b/analogvnn/graph/ModelGraphState.py index e7a3790..581de31 100644 --- a/analogvnn/graph/ModelGraphState.py +++ b/analogvnn/graph/ModelGraphState.py @@ -52,16 +52,18 @@ def __init__(self, use_autograd_graph: bool = False, allow_loops=False): self.forward_input_output_graph = None self._loss = None - # noinspection PyUnusedLocal - @staticmethod - def ready_for_forward(exception: bool = False) -> bool: + # noinspection PyUnusedLocal,PyMethodMayBeStatic + def ready_for_forward(self, exception: bool = False) -> bool: """Check if the state is ready for forward pass. Args: - exception (bool): if True, raise an exception if the state is not ready for forward pass. + exception (bool): If True, an exception is raised if the state is not ready for forward pass. Returns: bool: True if the state is ready for forward pass. + + Raises: + RuntimeError: If the state is not ready for forward pass and exception is True. """ return True From 624752186f3840da959db004f4896ed634546302 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 17:36:46 -0500 Subject: [PATCH 06/16] to version 1.0.0rc6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3cfcf69..1f124b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ name = "analogvnn" [project] # $ pip install analogvnn name = "analogvnn" -version = "1.0.0rc5" +version = "1.0.0rc6" description = "A fully modular framework for modeling and optimizing analog/photonic neural networks" # Optional readme = "README.md" requires-python = ">=3.7" From 6b96543ac08ad2bc036e754e8a36a6767e532e46 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 18:07:11 -0500 Subject: [PATCH 07/16] use cpu by default --- analogvnn/utils/is_cpu_cuda.py | 43 ++++++++++++++++++++++--------- docs/_static/AnalogVNN_Demo.ipynb | 1 + sample_code.py | 1 + sample_code_with_logs.py | 1 + 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/analogvnn/utils/is_cpu_cuda.py b/analogvnn/utils/is_cpu_cuda.py index 9c204fc..c5771d3 100644 --- a/analogvnn/utils/is_cpu_cuda.py +++ b/analogvnn/utils/is_cpu_cuda.py @@ -23,15 +23,25 @@ def __init__(self): super(CPUCuda, self).__init__() self._device = None self.device_name = None - self.reset_device() + self.use_cpu() - def reset_device(self): - """Reset the device to the default device. + def use_cpu(self) -> CPUCuda: + """Use cpu. Returns: CPUCuda: self """ - self.set_device(f'cuda:{torch.cuda.current_device()}' if torch.cuda.is_available() else 'cpu') + self.set_device('cpu') + return self + + def use_cuda_if_available(self) -> CPUCuda: + """Use cuda if available. + + Returns: + CPUCuda: self + """ + if torch.cuda.is_available(): + self.set_device(f'cuda:{torch.cuda.current_device()}') return self def set_device(self, device_name: str) -> CPUCuda: @@ -48,22 +58,31 @@ def set_device(self, device_name: str) -> CPUCuda: return self @property - def is_cuda(self) -> bool: - """Check if the device is cuda. + def device(self) -> torch.device: + """Get the device. Returns: - bool: True if the device is cuda, False otherwise. + torch.device: the device. """ - return 'cuda' in self.device_name + return self._device @property - def device(self) -> torch.device: - """Get the device. + def is_cpu(self) -> bool: + """Check if the device is cpu. Returns: - torch.device: the device. + bool: True if the device is cpu, False otherwise. """ - return self._device + return self.device_name.startswith('cpu') + + @property + def is_cuda(self) -> bool: + """Check if the device is cuda. + + Returns: + bool: True if the device is cuda, False otherwise. + """ + return self.device_name.startswith('cuda') @property def is_using_cuda(self) -> Tuple[torch.device, bool]: diff --git a/docs/_static/AnalogVNN_Demo.ipynb b/docs/_static/AnalogVNN_Demo.ipynb index ee559f3..09adbe0 100644 --- a/docs/_static/AnalogVNN_Demo.ipynb +++ b/docs/_static/AnalogVNN_Demo.ipynb @@ -118,6 +118,7 @@ "print(f\"AnalogVNN version: {analogvnn.__version__}\")\n", "print(f\"PyTorch version: {torch.__version__}\")\n", "\n", + "is_cpu_cuda.use_cuda_if_available()\n", "torch.manual_seed(0)\n", "device, is_cuda = is_cpu_cuda.is_using_cuda\n", "print(f\"Device: {device}\")" diff --git a/sample_code.py b/sample_code.py index 020437d..050959e 100644 --- a/sample_code.py +++ b/sample_code.py @@ -150,6 +150,7 @@ def __init__(self, norm_class, precision_class, precision, noise_class, leakage) def run_linear3_model(): """The main function to train photonics image classifier with 3 linear/dense nn for MNIST dataset.""" + is_cpu_cuda.use_cuda_if_available() torch.backends.cudnn.benchmark = True torch.manual_seed(0) device, is_cuda = is_cpu_cuda.is_using_cuda diff --git a/sample_code_with_logs.py b/sample_code_with_logs.py index ea64f2b..23ffdce 100644 --- a/sample_code_with_logs.py +++ b/sample_code_with_logs.py @@ -154,6 +154,7 @@ def __init__(self, norm_class, precision_class, precision, noise_class, leakage) def run_linear3_model(): """The main function to train photonics image classifier with 3 linear/dense nn for MNIST dataset.""" + is_cpu_cuda.use_cuda_if_available() torch.backends.cudnn.benchmark = True torch.manual_seed(0) data_path = Path('_data') From 94bd8b75179747095569bae953917a129c4b211c Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 21:09:29 -0500 Subject: [PATCH 08/16] added registered_modules --- analogvnn/nn/module/Layer.py | 53 +++++++++++++++++++++++++++++++++- analogvnn/nn/module/Model.py | 56 +++++++++++++++++++++++++++--------- 2 files changed, 95 insertions(+), 14 deletions(-) diff --git a/analogvnn/nn/module/Layer.py b/analogvnn/nn/module/Layer.py index e0a07c2..846144f 100644 --- a/analogvnn/nn/module/Layer.py +++ b/analogvnn/nn/module/Layer.py @@ -1,7 +1,7 @@ from __future__ import annotations import functools -from typing import Union, Type, Callable, Sequence, Optional, TYPE_CHECKING +from typing import Union, Type, Callable, Sequence, Optional, TYPE_CHECKING, Set, Iterator, Tuple from torch import nn, Tensor @@ -191,6 +191,57 @@ def set_backward_function(self, backward_class: Union[Callable, BackwardModule, return self + def named_registered_modules( + self, + memo: Optional[Set[nn.Module]] = None, + prefix: str = '', + remove_duplicate: bool = True + ) -> Iterator[Tuple[str, nn.Module]]: + """Returns an iterator over all registered modules in the network, yielding + both the name of the module and the module itself. + + Args: + memo: a memo to store the set of modules already added to the result + prefix: a prefix that will be added to the name of the module + remove_duplicate: whether to remove the duplicated module instances in the result + or not + + Yields: + (str, Module): Tuple of name and module + + Note: + Duplicate modules are returned only once. In the following + example, ``l`` will be returned only once. + """ + if memo is None: + memo = set() + + if self.backward_function != self: + memo.add(self.backward_function) + + for name, module in super(Layer, self).named_modules( + memo=memo, + prefix=prefix, + remove_duplicate=remove_duplicate + ): + if module is self: + continue + + yield name, module + + def registered_modules(self) -> Iterator[nn.Module]: + """Returns an iterator over registered modules under self. + + Yields: + nn.Module: a module in the network + + Note: + Duplicate modules are returned only once. In the following + example, ``l`` will be returned only once. + """ + for _, module in self.named_registered_modules(): + yield module + def _forward_wrapper(self, function: Callable) -> Callable: """Wrapper for the forward function. diff --git a/analogvnn/nn/module/Model.py b/analogvnn/nn/module/Model.py index fb40260..ca5e1a2 100644 --- a/analogvnn/nn/module/Model.py +++ b/analogvnn/nn/module/Model.py @@ -1,10 +1,10 @@ from __future__ import annotations import typing -from typing import Callable, Optional, Tuple +from typing import Callable, Optional, Tuple, Set, Iterator import torch -from torch import optim, Tensor +from torch import optim, Tensor, nn from torch.utils.data import DataLoader from analogvnn.fn.test import test @@ -76,6 +76,24 @@ def __init__(self, tensorboard_log_dir=None, device=is_cpu_cuda.device): self.accuracy_function = None self.device = device + def __call__(self, *args, **kwargs): + """Call the model. + + Args: + *args: The arguments of the model. + **kwargs: The keyword arguments of the model. + + Returns: + TENSORS: The output of the model. + + Raises: + RuntimeError: if the model is not compiled. + """ + if not self._compiled: + raise RuntimeError('Model is not compiled yet.') + + return super(Model, self).__call__(*args, **kwargs) + @property def use_autograd_graph(self): """Is the autograd graph used for the model. @@ -94,23 +112,35 @@ def use_autograd_graph(self, use_autograd_graph: bool): """ self.graphs.use_autograd_graph = use_autograd_graph - def __call__(self, *args, **kwargs): - """Call the model. + def named_registered_modules( + self, + memo: Optional[Set[nn.Module]] = None, + prefix: str = '', + remove_duplicate: bool = True + ) -> Iterator[Tuple[str, nn.Module]]: + """Returns an iterator over registered modules under self, + yielding both the name of the module and the module itself. Args: - *args: The arguments of the model. - **kwargs: The keyword arguments of the model. + memo: a memo to store the set of modules already added to the result + prefix: a prefix that will be added to the name of the module + remove_duplicate: whether to remove the duplicated module instances in the result + or not - Returns: - TENSORS: The output of the model. + Yields: + (str, nn.Module): Tuple of name and module - Raises: - RuntimeError: if the model is not compiled. + Note: + Duplicate modules are returned only once. In the following + example, ``l`` will be returned only once. """ - if not self._compiled: - raise RuntimeError('Model is not compiled yet.') + if memo is None: + memo = set() - return super(Model, self).__call__(*args, **kwargs) + memo.add(self.optimizer) + memo.add(self.loss_function) + memo.add(self.accuracy_function) + return super(Model, self).named_registered_modules(memo=memo, prefix=prefix, remove_duplicate=remove_duplicate) def compile(self, device: Optional[torch.device] = None, layer_data: bool = True): """Compile the model. From f29a191b0d6f11b794b3581af05ebc5f4081f303 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 21:09:55 -0500 Subject: [PATCH 09/16] simplified using registered_modules --- analogvnn/nn/module/FullSequential.py | 2 +- analogvnn/nn/module/Sequential.py | 131 ++++---------------------- 2 files changed, 17 insertions(+), 116 deletions(-) diff --git a/analogvnn/nn/module/FullSequential.py b/analogvnn/nn/module/FullSequential.py index 0c7e3a0..e31c795 100644 --- a/analogvnn/nn/module/FullSequential.py +++ b/analogvnn/nn/module/FullSequential.py @@ -22,6 +22,6 @@ def compile(self, device: Optional[torch.device] = None, layer_data: bool = True Returns: FullSequential: self """ - arr = [self.graphs.INPUT, *list(self._runtime_module_list.values()), self.graphs.OUTPUT] + arr = [self.graphs.INPUT, *list(self.registered_modules()), self.graphs.OUTPUT] self.graphs.backward_graph.add_connection(*reversed(arr)) return super().compile(device, layer_data) diff --git a/analogvnn/nn/module/Sequential.py b/analogvnn/nn/module/Sequential.py index 1ec0e67..ca3d5b4 100644 --- a/analogvnn/nn/module/Sequential.py +++ b/analogvnn/nn/module/Sequential.py @@ -1,13 +1,9 @@ from __future__ import annotations -import operator -from collections import OrderedDict -from itertools import islice -from typing import Iterator, TypeVar, Union, Dict, Optional +from typing import TypeVar, Optional import torch from torch import nn -from torch.nn import Module from analogvnn.nn.module.Model import Model @@ -16,22 +12,23 @@ __all__ = ['Sequential'] -class Sequential(Model): - """A sequential model. +class Sequential(Model, nn.Sequential): + """Base class for all sequential models.""" - Attributes: - _runtime_module_list (OrderedDict[str, nn.Module]): The ordered dictionary of the modules. - """ - - def __init__(self, *args): - """Initialize the model. + def __call__(self, *args, **kwargs): + """Call the model. Args: - *args: The modules to add. + *args: The input. + **kwargs: The input. + + Returns: + torch.Tensor: The output of the model. """ - super(Sequential, self).__init__() - self._runtime_module_list: Dict[str, Optional[Module]] = OrderedDict() - self.add_sequence(*args) + if not self._compiled: + self.compile() + + return super().__call__(*args, **kwargs) def compile(self, device: Optional[torch.device] = None, layer_data: bool = True): """Compile the model and add forward graph. @@ -43,7 +40,7 @@ def compile(self, device: Optional[torch.device] = None, layer_data: bool = True Returns: Sequential: self """ - arr = [self.graphs.INPUT, *list(self._runtime_module_list.values()), self.graphs.OUTPUT] + arr = [self.graphs.INPUT, *list(self.registered_modules()), self.graphs.OUTPUT] self.graphs.forward_graph.add_connection(*arr) return super().compile(device, layer_data) @@ -53,100 +50,4 @@ def add_sequence(self, *args): Args: *args (nn.Module): The modules to add. """ - if len(args) == 1 and isinstance(args[0], OrderedDict): - for key, module in args[0].items(): - self._add_run_module(key, module) - else: - for idx, module in enumerate(args): - self._add_run_module(str(idx), module) - - def _add_run_module(self, name: str, module: Optional[Module]): - """Add a module to the forward graph of model. - - Args: - name (str): The name of the module. - module (nn.Module): The module to add. - """ - self.add_module(name, module) - self._runtime_module_list[name] = module - return self - - def _get_item_by_idx(self, iterator, idx) -> T: - """Get the idx-th item of the iterator. - - Args: - iterator (Iterator): The iterator. - idx (int): The index of the item. - - Returns: - T: The idx-th item of the iterator. - """ - size = len(self) - idx = operator.index(idx) - if not -size <= idx < size: - raise IndexError('index {} is out of range'.format(idx)) - idx %= size - return next(islice(iterator, idx, None)) - - def __getitem__(self, idx) -> Union[Sequential, T]: - """Get the idx-th module of the model. - - Args: - idx (int): The index of the module. - - Returns: - Union[Sequential, T]: The idx-th module of the model. - """ - if isinstance(idx, slice): - return self.__class__(OrderedDict(list(self._runtime_module_list.items())[idx])) - else: - return self._get_item_by_idx(self._runtime_module_list.values(), idx) - - def __setitem__(self, idx: int, module: nn.Module) -> None: - """Set the idx-th module of the model. - - Args: - idx (int): The index of the module. - module (nn.Module): The module to set. - """ - key: str = self._get_item_by_idx(self._runtime_module_list.keys(), idx) - return setattr(self, key, module) - - def __delitem__(self, idx: Union[slice, int]) -> None: - """Remove the idx-th module from the model. - - Args: - idx (Union[slice, int]): The index of the module. - """ - if isinstance(idx, slice): - for key in list(self._runtime_module_list.keys())[idx]: - delattr(self, key) - else: - key = self._get_item_by_idx(self._runtime_module_list.keys(), idx) - delattr(self, key) - - def __len__(self) -> int: - """Return the number of modules in the model. - - Returns: - int: The number of modules in the model. - """ - return len(self._runtime_module_list) - - def __dir__(self): - """Return the list of attributes of the module. - - Returns: - list: The list of attributes of the module. - """ - keys = super(Sequential, self).__dir__() - keys = [key for key in keys if not key.isdigit()] - return keys - - def __iter__(self) -> Iterator[nn.Module]: - """Return an iterator over the modules of the model. - - Returns: - Iterator[nn.Module]: An iterator over the modules of the model. - """ - return iter(self._runtime_module_list.values()) + return self.extend(args) From 0a603395283480a41865614bb1e2ed5528a24039 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 22:50:53 -0500 Subject: [PATCH 10/16] updated importlib_metadata --- analogvnn/__init__.py | 12 ++++++++---- pyproject.toml | 5 ++++- requirements.txt | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/analogvnn/__init__.py b/analogvnn/__init__.py index 32f5812..fb54294 100644 --- a/analogvnn/__init__.py +++ b/analogvnn/__init__.py @@ -1,14 +1,18 @@ import sys -import importlib_metadata +if sys.version_info[:2] >= (3, 8): + from importlib import metadata +else: + import importlib_metadata as metadata # pragma: no cover + __package__ = 'analogvnn' __author__ = 'Vivswan Shah (vivswanshah@pitt.edu)' try: - __version__ = importlib_metadata.version(__package__) -except importlib_metadata.PackageNotFoundError: - __version__ = 'local' + __version__ = metadata.version(__package__) +except metadata.PackageNotFoundError: + __version__ = '0.0.0' if sys.version_info < (3, 7, 0): import warnings diff --git a/pyproject.toml b/pyproject.toml index 1f124b0..0f24fc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ dependencies = [ "scipy", "numpy>=1.16.5", "networkx", - "importlib_metadata", + "importlib-metadata<5.0.0,>=2.0.0; python_version < '3.8'", ] # List additional groups of dependencies here (e.g. development @@ -101,6 +101,9 @@ flake8 = [ "flake8-comprehensions", "flake8-executable", "flake8-coding", + "flake8-return", + "flake8-noreturn", + "flake8-deprecated", ] dev = [ "setuptools>=61.0.0", diff --git a/requirements.txt b/requirements.txt index 99fb2b1..d5668b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ torchaudio numpy scipy networkx -importlib_metadata +importlib-metadata<5.0.0,>=2.0.0; python_version < "3.8" # Full tensorflow From ab253a5dad5c13c3cd134e31966cef171d22e690 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 22:51:41 -0500 Subject: [PATCH 11/16] fixed return value --- analogvnn/graph/ModelGraphState.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/analogvnn/graph/ModelGraphState.py b/analogvnn/graph/ModelGraphState.py index 581de31..9021cab 100644 --- a/analogvnn/graph/ModelGraphState.py +++ b/analogvnn/graph/ModelGraphState.py @@ -45,6 +45,7 @@ def __init__(self, use_autograd_graph: bool = False, allow_loops=False): use_autograd_graph: If True, the autograd graph is used to calculate the gradients. allow_loops: If True, the graph is allowed to contain loops. """ + super(ModelGraphState, self).__init__() self.allow_loops = allow_loops self.use_autograd_graph = use_autograd_graph @@ -65,6 +66,7 @@ def ready_for_forward(self, exception: bool = False) -> bool: Raises: RuntimeError: If the state is not ready for forward pass and exception is True. """ + return True def ready_for_backward(self, exception: bool = False) -> bool: @@ -79,14 +81,15 @@ def ready_for_backward(self, exception: bool = False) -> bool: Raises: RuntimeError: if the state is not ready for backward pass and exception is True. """ + if exception: if self.outputs is None: raise RuntimeError('output is not set.') if self._loss is None: raise RuntimeError('loss is not set.') - else: - return not (self.outputs is None or self._loss is None) + + return not (self.outputs is None or self._loss is None) @property def inputs(self) -> Optional[ArgsKwargs]: @@ -95,6 +98,7 @@ def inputs(self) -> Optional[ArgsKwargs]: Returns: ArgsKwargs: the inputs. """ + if self.INPUT not in self.forward_input_output_graph: return None return self.forward_input_output_graph[self.INPUT].inputs @@ -106,6 +110,7 @@ def outputs(self) -> Optional[ArgsKwargs]: Returns: ArgsKwargs: the output. """ + if self.OUTPUT not in self.forward_input_output_graph: return None return self.forward_input_output_graph[self.OUTPUT].outputs @@ -117,6 +122,7 @@ def loss(self): Returns: Tensor: the loss. """ + return self._loss def set_loss(self, loss: Union[Tensor, None]) -> ModelGraphState: @@ -128,5 +134,6 @@ def set_loss(self, loss: Union[Tensor, None]) -> ModelGraphState: Returns: ModelGraphState: self. """ + self._loss = loss return self From ba9ca6dc80554805d2d43cecebc7ade3fb19652d Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 23:13:16 -0500 Subject: [PATCH 12/16] flake8: inverse D202 Signed-off-by: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> --- .flake8 | 3 + .readthedocs.yaml | 10 +- analogvnn/backward/BackwardFunction.py | 5 + analogvnn/backward/BackwardIdentity.py | 1 + analogvnn/backward/BackwardModule.py | 14 +++ analogvnn/backward/BackwardUsingForward.py | 1 + analogvnn/fn/dirac_delta.py | 1 + analogvnn/fn/reduce_precision.py | 2 + analogvnn/fn/test.py | 1 + analogvnn/fn/to_matrix.py | 1 + analogvnn/fn/train.py | 1 + analogvnn/graph/AccumulateGrad.py | 3 + analogvnn/graph/AcyclicDirectedGraph.py | 15 ++- analogvnn/graph/ArgsKwargs.py | 5 + analogvnn/graph/BackwardGraph.py | 6 ++ analogvnn/graph/ForwardGraph.py | 5 + analogvnn/graph/ModelGraph.py | 2 + analogvnn/graph/to_graph_viz_digraph.py | 1 + analogvnn/nn/Linear.py | 5 + analogvnn/nn/activation/Activation.py | 2 + analogvnn/nn/activation/BinaryStep.py | 2 + analogvnn/nn/activation/ELU.py | 6 ++ analogvnn/nn/activation/Gaussian.py | 4 + analogvnn/nn/activation/Identity.py | 4 + analogvnn/nn/activation/ReLU.py | 9 ++ analogvnn/nn/activation/SiLU.py | 2 + analogvnn/nn/activation/Sigmoid.py | 4 + analogvnn/nn/activation/Tanh.py | 4 + analogvnn/nn/module/FullSequential.py | 1 + analogvnn/nn/module/Layer.py | 22 ++++- analogvnn/nn/module/Model.py | 17 +++- analogvnn/nn/module/Sequential.py | 3 + analogvnn/nn/noise/GaussianNoise.py | 12 +++ analogvnn/nn/noise/LaplacianNoise.py | 12 +++ analogvnn/nn/noise/PoissonNoise.py | 13 +++ analogvnn/nn/noise/UniformNoise.py | 12 +++ analogvnn/nn/normalize/Clamp.py | 4 + analogvnn/nn/normalize/LPNorm.py | 11 +++ analogvnn/nn/precision/ReducePrecision.py | 6 ++ .../nn/precision/StochasticReducePrecision.py | 6 ++ analogvnn/parameter/Parameter.py | 3 + analogvnn/parameter/PseudoParameter.py | 14 +++ analogvnn/utils/TensorboardModelLog.py | 10 ++ analogvnn/utils/is_cpu_cuda.py | 9 ++ analogvnn/utils/render_autograd_graph.py | 31 ++++++ analogvnn/utils/to_tensor_parameter.py | 2 + docs/_static/analogvnn-logo-square-black.svg | 60 ++++++------ docs/_static/analogvnn-logo-square-white.svg | 60 ++++++------ docs/_static/analogvnn-logo-wide-black.svg | 95 +++++++++++-------- docs/_static/analogvnn-logo-wide-white.svg | 95 +++++++++++-------- docs/_static/custom.css | 2 +- docs/extra_classes.md | 1 + docs/inner_workings.md | 9 +- docs/install.md | 18 ++-- requirements-test.txt | 3 + sample_code.py | 7 ++ sample_code_with_logs.py | 6 ++ unit_tests/test_model_graphs.py | 4 + unit_tests/test_pseudo_parameter.py | 3 + 59 files changed, 504 insertions(+), 166 deletions(-) diff --git a/.flake8 b/.flake8 index 2abb4db..4f02f5c 100644 --- a/.flake8 +++ b/.flake8 @@ -5,7 +5,10 @@ extend-ignore = C101, # Coding magic comment D100, # Missing docstring in public module D104, # Missing docstring in public package + D202, # No blank lines allowed after function docstring D401, # First line should be in imperative mood + R504, # unnecessary variable assignment before return statement + R505, # unnecessary else after return statement per-file-ignores = sample_code.py: D100, D101, D102, D103, D104 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f513a67..395798e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,8 +19,8 @@ sphinx: formats: all # Optionally declare the Python requirements required to build your docs python: - install: - - method: pip - path: . - extra_requirements: - - doc + install: + - method: pip + path: . + extra_requirements: + - doc diff --git a/analogvnn/backward/BackwardFunction.py b/analogvnn/backward/BackwardFunction.py index 0760003..73fc7b5 100644 --- a/analogvnn/backward/BackwardFunction.py +++ b/analogvnn/backward/BackwardFunction.py @@ -27,6 +27,7 @@ def __init__(self, backward_function: Callable, layer: nn.Module = None): backward_function (Callable): The function used to compute the backward gradient. layer (nn.Module): The layer that this backward module is associated with. """ + super(BackwardFunction, self).__init__(layer) self._backward_function = backward_function @@ -37,6 +38,7 @@ def backward_function(self) -> Callable: Returns: Callable: The function used to compute the backward gradient. """ + return self._backward_function @backward_function.setter @@ -46,6 +48,7 @@ def backward_function(self, backward_function: Callable): Args: backward_function (Callable): The function used to compute the backward gradient with. """ + self.set_backward_function(backward_function) def set_backward_function(self, backward_function: Callable) -> BackwardFunction: @@ -57,6 +60,7 @@ def set_backward_function(self, backward_function: Callable) -> BackwardFunction Returns: BackwardFunction: self. """ + self._backward_function = backward_function return self @@ -73,6 +77,7 @@ def backward(self, *grad_output: Tensor, **grad_output_kwarg: Tensor) -> TENSORS Raises: NotImplementedError: If the backward function is not set. """ + if self._backward_function is None: raise ValueError('set backward_function first before invoking backward') diff --git a/analogvnn/backward/BackwardIdentity.py b/analogvnn/backward/BackwardIdentity.py index f8c9e61..96006bc 100644 --- a/analogvnn/backward/BackwardIdentity.py +++ b/analogvnn/backward/BackwardIdentity.py @@ -21,6 +21,7 @@ def backward(self, *grad_output: Tensor, **grad_output_kwarg: Tensor) -> TENSORS Returns: TENSORS: The gradients of the input of the layer. """ + if len(grad_output) == 0: return None diff --git a/analogvnn/backward/BackwardModule.py b/analogvnn/backward/BackwardModule.py index ed1fccf..6193a12 100644 --- a/analogvnn/backward/BackwardModule.py +++ b/analogvnn/backward/BackwardModule.py @@ -50,6 +50,7 @@ def forward(ctx: Any, backward_module: BackwardModule, _: Tensor, *args: Tensor, Returns: TENSORS: The output of the function. """ + ctx.backward_module = backward_module return ctx.backward_module._call_impl_forward(*args, **kwargs) @@ -64,6 +65,7 @@ def backward(ctx: Any, *grad_outputs: Tensor) -> Tuple[None, None, TENSORS]: Returns: TENSORS: The gradients of the input of the function. """ + backward_module: BackwardModule = ctx.backward_module results = backward_module._call_impl_backward(*grad_outputs) @@ -78,6 +80,7 @@ def __init__(self, layer: nn.Module = None): Args: layer (nn.Module): The layer for which the backward gradient is computed. """ + super(BackwardModule, self).__init__() self._layer = None self._set_autograd_backward() @@ -97,6 +100,7 @@ def forward(self, *inputs: Tensor, **inputs_kwarg: Tensor) -> TENSORS: Raises: NotImplementedError: If the forward pass is not implemented. """ + raise NotImplementedError(f'Module [{type(self).__name__}] is missing the required "forward" function') forward._implemented = False @@ -115,6 +119,7 @@ def backward(self, *grad_outputs: Tensor, **grad_output_kwarg: Tensor) -> TENSOR Raises: NotImplementedError: If the backward pass is not implemented. """ + raise NotImplementedError(f'Module [{type(self).__name__}] is missing the required "backward" function') def _call_impl_forward(self, *args: Tensor, **kwarg: Tensor) -> TENSORS: @@ -127,6 +132,7 @@ def _call_impl_forward(self, *args: Tensor, **kwarg: Tensor) -> TENSORS: Returns: TENSORS: The output of the layer. """ + return self.forward(*args, **kwarg) def _call_impl_backward(self, *grad_output: Tensor, **grad_output_kwarg: Tensor) -> TENSORS: @@ -139,6 +145,7 @@ def _call_impl_backward(self, *grad_output: Tensor, **grad_output_kwarg: Tensor) Returns: TENSORS: The gradients of the input of the layer. """ + return self.backward(*grad_output, **grad_output_kwarg) __call__: Callable[..., Any] = _call_impl_backward @@ -155,6 +162,7 @@ def auto_apply(self, *args: Tensor, to_apply=True, **kwargs: Tensor) -> TENSORS: Returns: TENSORS: The output of the layer. """ + if to_apply and not self._disable_autograd_backward: return self._autograd_backward.apply(self, self._empty_holder_tensor, *args, **kwargs) else: @@ -166,6 +174,7 @@ def has_forward(self) -> bool: Returns: bool: True if the forward pass is implemented, False otherwise. """ + return not hasattr(self.forward, '_implemented') @property @@ -175,6 +184,7 @@ def layer(self) -> Optional[nn.Module]: Returns: Optional[nn.Module]: layer """ + return self.get_layer() def get_layer(self) -> Optional[nn.Module]: @@ -183,6 +193,7 @@ def get_layer(self) -> Optional[nn.Module]: Returns: Optional[nn.Module]: layer """ + if isinstance(self, nn.Module): return self else: @@ -202,6 +213,7 @@ def set_layer(self, layer: Optional[nn.Module]) -> BackwardModule: ValueError: If the layer is already set. ValueError: If the layer is not an instance of nn.Module. """ + if isinstance(self, nn.Module): raise ValueError('layer of Backward Module is set to itself') if self._layer is not None: @@ -237,6 +249,7 @@ def set_grad_of(tensor: Tensor, grad: Tensor) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the tensor. """ + if tensor is None or tensor.requires_grad is False: return None @@ -267,6 +280,7 @@ def __getattr__(self, name: str) -> Any: Raises: AttributeError: If the attribute is not found. """ + if isinstance(self, nn.Module) or self == self._layer: return super(BackwardModule, self).__getattr__(name) if not str(name).startswith('__') and self._layer is not None and hasattr(self._layer, name): diff --git a/analogvnn/backward/BackwardUsingForward.py b/analogvnn/backward/BackwardUsingForward.py index 819eff7..95338d0 100644 --- a/analogvnn/backward/BackwardUsingForward.py +++ b/analogvnn/backward/BackwardUsingForward.py @@ -21,4 +21,5 @@ def backward(self, *grad_output: Tensor, **grad_output_kwarg: Tensor) -> TENSORS Returns: TENSORS: The gradients of the input of the layer. """ + return self._layer.forward(*grad_output, **grad_output_kwarg) diff --git a/analogvnn/fn/dirac_delta.py b/analogvnn/fn/dirac_delta.py index ca16db2..8e9f021 100644 --- a/analogvnn/fn/dirac_delta.py +++ b/analogvnn/fn/dirac_delta.py @@ -16,4 +16,5 @@ def dirac_delta(x: TENSOR_OPERABLE, a: TENSOR_OPERABLE = 0.001) -> TENSOR_OPERAB TENSOR_OPERABLE: TENSOR_OPERABLE with the same shape as x, but with values equal to the Dirac delta function of x. """ + return 1 / (np.abs(a) * np.sqrt(np.pi)) * np.exp(-((x / a) ** 2)) diff --git a/analogvnn/fn/reduce_precision.py b/analogvnn/fn/reduce_precision.py index 2d7ccc3..1c8f394 100644 --- a/analogvnn/fn/reduce_precision.py +++ b/analogvnn/fn/reduce_precision.py @@ -18,6 +18,7 @@ def reduce_precision(x: TENSOR_OPERABLE, precision: TENSOR_OPERABLE, divide: TEN TENSOR_OPERABLE: TENSOR_OPERABLE with the same shape as x, but with values rounded to the nearest multiple of precision. """ + x = x if isinstance(x, Tensor) else torch.tensor(x, requires_grad=False) g: Tensor = x * precision f = torch.sign(g) * torch.maximum( @@ -38,6 +39,7 @@ def stochastic_reduce_precision(x: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) TENSOR_OPERABLE: TENSOR_OPERABLE with the same shape as x, but with values rounded to the nearest multiple of precision. """ + g: Tensor = x * precision rand_x = torch.rand_like(g, requires_grad=False) diff --git a/analogvnn/fn/test.py b/analogvnn/fn/test.py index c4496cf..a2087a3 100644 --- a/analogvnn/fn/test.py +++ b/analogvnn/fn/test.py @@ -18,6 +18,7 @@ def test(model: nn.Module, test_loader: DataLoader, test_run: bool = False) -> T Returns: tuple: the loss and accuracy of the model on the test set. """ + model.eval() total_loss = 0 total_accuracy = 0 diff --git a/analogvnn/fn/to_matrix.py b/analogvnn/fn/to_matrix.py index facd900..392b0ff 100644 --- a/analogvnn/fn/to_matrix.py +++ b/analogvnn/fn/to_matrix.py @@ -12,6 +12,7 @@ def to_matrix(tensor: Tensor) -> Tensor: Returns: Tensor: Tensor with the same values as the tensor, but with shape (1, -1). """ + if len(tensor.size()) == 1: return tensor.view(1, -1) return tensor diff --git a/analogvnn/fn/train.py b/analogvnn/fn/train.py index 2b32b7f..5de064c 100644 --- a/analogvnn/fn/train.py +++ b/analogvnn/fn/train.py @@ -23,6 +23,7 @@ def train( Returns: tuple: the loss and accuracy of the model on the train set. """ + model.train() total_loss = 0.0 total_accuracy = 0 diff --git a/analogvnn/graph/AccumulateGrad.py b/analogvnn/graph/AccumulateGrad.py index d46b9c9..4451250 100644 --- a/analogvnn/graph/AccumulateGrad.py +++ b/analogvnn/graph/AccumulateGrad.py @@ -29,6 +29,7 @@ def __init__(self, module: Union[nn.Module, Callable]): Args: module (Union[nn.Module, Callable]): Module from which to accumulate gradients. """ + super(AccumulateGrad, self).__init__() self.input_output_connections = {} self.module = module @@ -39,6 +40,7 @@ def __repr__(self): Returns: str: String representation of the module. """ + return f'AccumulateGrad({self.module})' def __call__( # noqa: C901 @@ -55,6 +57,7 @@ def __call__( # noqa: C901 Returns: ArgsKwargs: The output gradients. """ + grad_inputs_args = {} grad_inputs_kwargs = {} for key, grad_output in grad_outputs_args_kwargs.kwargs.items(): diff --git a/analogvnn/graph/AcyclicDirectedGraph.py b/analogvnn/graph/AcyclicDirectedGraph.py index 3477a8d..7f14b19 100644 --- a/analogvnn/graph/AcyclicDirectedGraph.py +++ b/analogvnn/graph/AcyclicDirectedGraph.py @@ -49,6 +49,7 @@ def __init__(self, graph_state: ModelGraphState = None): Raises: NotImplementedError: If allow_loops is True, since this is not implemented yet. """ + super(AcyclicDirectedGraph, self).__init__() self.graph = nx.MultiDiGraph() self.graph_state = graph_state @@ -69,6 +70,7 @@ def __call__(self, *args, **kwargs): Raises: NotImplementedError: since method is abstract """ + raise NotImplementedError def add_connection(self, *args: GRAPH_NODE_TYPE): @@ -80,6 +82,7 @@ def add_connection(self, *args: GRAPH_NODE_TYPE): Returns: AcyclicDirectedGraph: self. """ + for i in range(1, len(args)): self.add_edge(args[i - 1], args[i]) return self @@ -106,6 +109,7 @@ def add_edge( Returns: AcyclicDirectedGraph: self. """ + attr = self.check_edge_parameters(in_arg, in_kwarg, out_arg, out_kwarg) existing_edges = self.graph.get_edge_data(u_of_edge, v_of_edge) @@ -127,8 +131,8 @@ def add_edge( self.graph.nodes[v_of_edge]['fillcolor'] = 'lightblue' return self - @staticmethod # noqa: C901 - def check_edge_parameters( + @staticmethod + def check_edge_parameters( # noqa: C901 in_arg: Union[None, int, bool], in_kwarg: Union[None, str, bool], out_arg: Union[None, int, bool], @@ -148,6 +152,7 @@ def check_edge_parameters( Raises: ValueError: If in and out parameters are invalid. """ + # @@@ in_arg: None in_kwarg: None out_arg: None out_kwarg: None 0 # @@ in_arg: True in_kwarg: None out_arg: True out_kwarg: None 1 # in_arg: None in_kwarg: True out_arg: True out_kwarg: None 2 @@ -237,6 +242,7 @@ def _create_edge_label( Returns: str: The edge's label. """ + label = '' if in_arg == in_kwarg == out_arg == out_kwarg is True: return '* -> *' @@ -274,6 +280,7 @@ def compile(self, is_static: bool = True): Raises: ValueError: If the graph is not acyclic. """ + for i in nx.simple_cycles(self.graph): raise ValueError(f'Cycle detected: {i}') @@ -292,6 +299,7 @@ def _reindex_out_args(graph: nx.MultiDiGraph) -> nx.MultiDiGraph: Returns: nx.MultiDiGraph: The graph with re-indexed output arguments. """ + # noinspection PyTypeChecker graph: nx.MultiDiGraph = graph.copy() @@ -324,6 +332,7 @@ def _create_static_sub_graph( Returns: List[Tuple[GRAPH_NODE_TYPE, List[GRAPH_NODE_TYPE]]]: The static sub graph. """ + if self._is_static and from_node in self._static_graphs: return self._static_graphs[from_node] @@ -358,6 +367,7 @@ def parse_args_kwargs( # noqa: C901 Returns: ArgsKwargs: The arguments and keyword arguments. """ + args = {} extra_args = [] kwargs = {} @@ -436,4 +446,5 @@ def render(self, *args, real_label: bool = False, **kwargs) -> str: Returns: str: The (possibly relative) path of the rendered file. """ + return to_graphviz_digraph(self.graph, real_label=real_label).render(*args, **kwargs) diff --git a/analogvnn/graph/ArgsKwargs.py b/analogvnn/graph/ArgsKwargs.py index 61c1116..0899ea4 100644 --- a/analogvnn/graph/ArgsKwargs.py +++ b/analogvnn/graph/ArgsKwargs.py @@ -38,6 +38,7 @@ def __init__(self, args=None, kwargs=None): args: The arguments. kwargs: The keyword arguments. """ + super(ArgsKwargs, self).__init__() if args is None: args = [] @@ -55,10 +56,12 @@ def __init__(self, args=None, kwargs=None): def is_empty(self): """Returns whether the ArgsKwargs object is empty.""" + return len(self.args) == 0 and not bool(self.kwargs) def __repr__(self): """Returns a string representation of the parameter.""" + return f'ArgsKwargs(args={self.args}, kwargs={self.kwargs})' @classmethod @@ -71,6 +74,7 @@ def to_args_kwargs_object(cls, outputs: ArgsKwargsInput) -> ArgsKwargs: Returns: ArgsKwargs: The ArgsKwargs object """ + if isinstance(outputs, cls): pass elif isinstance(outputs, dict): @@ -91,6 +95,7 @@ def from_args_kwargs_object(outputs: ArgsKwargs) -> ArgsKwargsOutput: Returns: ArgsKwargsOutput: object or tuple or dict """ + if len(outputs.kwargs.keys()) > 0: return outputs elif len(outputs.args) > 1: diff --git a/analogvnn/graph/BackwardGraph.py b/analogvnn/graph/BackwardGraph.py index 7aff94c..fe0813b 100644 --- a/analogvnn/graph/BackwardGraph.py +++ b/analogvnn/graph/BackwardGraph.py @@ -30,6 +30,7 @@ def __call__(self, gradient: TENSORS = None) -> ArgsKwargsOutput: Returns: ArgsKwargsOutput: gradient of the inputs function w.r.t. loss """ + self.graph_state.ready_for_backward(exception=True) if len(gradient) == 0: @@ -64,6 +65,7 @@ def compile(self, is_static=True): Raises: ValueError: If no forward pass has been performed yet. """ + if not self.graph.has_node(self.OUTPUT): raise ValueError("OUTPUT doesn't exist in the forward graph. Please preform a forward pass first.") @@ -78,6 +80,7 @@ def from_forward(self, forward_graph: Union[AcyclicDirectedGraph, nx.DiGraph]) - Returns: BackwardGraph: self. """ + if isinstance(forward_graph, AcyclicDirectedGraph): forward_graph = forward_graph.graph @@ -175,6 +178,7 @@ def calculate(self, *args, **kwargs) -> ArgsKwargsOutput: Raises: ValueError: If no forward pass has been performed yet. """ + if self.graph_state.forward_input_output_graph is None: raise ValueError('No forward pass has been performed yet. Please preform a forward pass first.') @@ -197,6 +201,7 @@ def _pass(self, from_node: GRAPH_NODE_TYPE, *args, **kwargs) -> Dict[GRAPH_NODE_ Returns: Dict[GRAPH_NODE_TYPE, InputOutput]: The input and output gradients of each node. """ + static_graph: List[Tuple[GRAPH_NODE_TYPE, List[GRAPH_NODE_TYPE]]] = self._create_static_sub_graph(from_node) input_output_graph: Dict[GRAPH_NODE_TYPE, InputOutput] = { from_node: InputOutput(inputs=ArgsKwargs( @@ -235,6 +240,7 @@ def _calculate_gradients( # noqa: C901 Returns: ArgsKwargs: The input gradients of the module. """ + if module in self.graph_state.forward_input_output_graph: module_inputs_outputs = self.graph_state.forward_input_output_graph[module] else: diff --git a/analogvnn/graph/ForwardGraph.py b/analogvnn/graph/ForwardGraph.py index 9dea69a..f66882c 100644 --- a/analogvnn/graph/ForwardGraph.py +++ b/analogvnn/graph/ForwardGraph.py @@ -26,6 +26,7 @@ def __call__(self, inputs: TENSORS, is_training: bool) -> ArgsKwargsOutput: Returns: ArgsKwargsOutput: Output of the graph """ + self.graph_state.ready_for_forward(exception=True) outputs = self.calculate(inputs, is_training) return outputs @@ -42,6 +43,7 @@ def compile(self, is_static: bool = True): Raises: ValueError: If no forward pass has been performed yet. """ + if not self.graph.has_node(self.INPUT): raise ValueError("INPUT doesn't exist in the forward graph. Please preform a forward pass first.") @@ -66,6 +68,7 @@ def calculate( Returns: ArgsKwargsOutput: Output of the graph """ + if not isinstance(inputs, Sequence): inputs = (inputs,) @@ -90,6 +93,7 @@ def _pass(self, from_node: GraphEnum, *inputs: Tensor) -> Dict[GraphEnum, InputO Returns: Dict[GraphEnum, InputOutput]: The input and output of each node """ + static_graph = self._create_static_sub_graph(from_node) input_output_graph = { from_node: InputOutput(inputs=ArgsKwargs(args=[*inputs])) @@ -124,6 +128,7 @@ def _detach_tensor(tensor: torch.Tensor) -> torch.Tensor: Returns: torch.Tensor: Detached tensor """ + tensor: torch.Tensor = tensor.detach() tensor.requires_grad = True return tensor diff --git a/analogvnn/graph/ModelGraph.py b/analogvnn/graph/ModelGraph.py index 57d7874..a14b5fd 100644 --- a/analogvnn/graph/ModelGraph.py +++ b/analogvnn/graph/ModelGraph.py @@ -25,6 +25,7 @@ def __init__(self, use_autograd_graph: bool = False, allow_loops: bool = False): use_autograd_graph: If True, the autograd graph is used to calculate the gradients. allow_loops: If True, the graph is allowed to contain loops. """ + super(ModelGraph, self).__init__(use_autograd_graph, allow_loops) self.forward_graph = ForwardGraph(self) self.backward_graph = BackwardGraph(self) @@ -39,6 +40,7 @@ def compile(self, is_static: bool = True, auto_backward_graph: bool = False) -> Returns: ModelGraph: self. """ + self.forward_graph.compile(is_static=is_static) if auto_backward_graph: diff --git a/analogvnn/graph/to_graph_viz_digraph.py b/analogvnn/graph/to_graph_viz_digraph.py index 96fdda7..9f9ffd3 100644 --- a/analogvnn/graph/to_graph_viz_digraph.py +++ b/analogvnn/graph/to_graph_viz_digraph.py @@ -26,6 +26,7 @@ def to_graphviz_digraph(from_graph: networkx.DiGraph, real_label: bool = False) ImportError: if pygraphviz (https://pygraphviz.github.io/) is not available. ImportError: if graphviz (https://pygraphviz.github.io/) is not available. """ + try: # noinspection PyPackageRequirements import pygraphviz # noqa: F401 diff --git a/analogvnn/nn/Linear.py b/analogvnn/nn/Linear.py index eceb7d3..445493e 100644 --- a/analogvnn/nn/Linear.py +++ b/analogvnn/nn/Linear.py @@ -23,6 +23,7 @@ def forward(self, x: Tensor): Returns: Tensor: The output of the linear layer. """ + if self.bias is not None: y = torch.addmm(self.bias, x, self.weight.t()) else: @@ -40,6 +41,7 @@ def backward(self, grad_output: Optional[Tensor], weight: Optional[Tensor] = Non Returns: Optional[Tensor]: The gradient of the input. """ + grad_output = to_matrix(grad_output) weight = to_matrix(self.weight if weight is None else weight) @@ -74,6 +76,7 @@ def __init__(self, in_features: int, out_features: int, bias: bool = True): out_features (int): The number of output features. bias (bool): True if the layer has a bias. """ + super(Linear, self).__init__() self.in_features = in_features self.out_features = out_features @@ -89,6 +92,7 @@ def __init__(self, in_features: int, out_features: int, bias: bool = True): def reset_parameters(self): """Reset the parameters of the layer.""" + nn.init.xavier_uniform_(self.weight) if self.bias is not None: fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.weight) @@ -101,4 +105,5 @@ def extra_repr(self) -> str: Returns: str: The extra representation of the linear layer. """ + return f'in_features={self.in_features}, out_features={self.out_features}, bias={self.bias is not None}' diff --git a/analogvnn/nn/activation/Activation.py b/analogvnn/nn/activation/Activation.py index 902194b..e779823 100644 --- a/analogvnn/nn/activation/Activation.py +++ b/analogvnn/nn/activation/Activation.py @@ -21,6 +21,7 @@ def initialise(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.xavier_uniform(tensor) @staticmethod @@ -33,6 +34,7 @@ def initialise_(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.xavier_uniform_(tensor) diff --git a/analogvnn/nn/activation/BinaryStep.py b/analogvnn/nn/activation/BinaryStep.py index b8684b2..9213b53 100644 --- a/analogvnn/nn/activation/BinaryStep.py +++ b/analogvnn/nn/activation/BinaryStep.py @@ -21,6 +21,7 @@ def forward(x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return (x >= 0).type(torch.float) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @@ -32,4 +33,5 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor. """ + return torch.zeros_like(grad_output) diff --git a/analogvnn/nn/activation/ELU.py b/analogvnn/nn/activation/ELU.py index e736a2c..03cabce 100644 --- a/analogvnn/nn/activation/ELU.py +++ b/analogvnn/nn/activation/ELU.py @@ -27,6 +27,7 @@ def __init__(self, alpha: float = 1.0507, scale_factor: float = 1.): alpha (float): the alpha parameter. scale_factor (float): the scale factor parameter. """ + super(SELU, self).__init__() self.alpha = nn.Parameter(torch.tensor(alpha), requires_grad=False) self.scale_factor = nn.Parameter(torch.tensor(scale_factor), requires_grad=False) @@ -40,6 +41,7 @@ def forward(self, x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return self.scale_factor * ( (x <= 0).type(torch.float) * self.alpha * (torch.exp(x) - 1) + (x > 0).type(torch.float) * x @@ -54,6 +56,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor. """ + x = self.inputs grad = self.scale_factor * ((x < 0).type(torch.float) * self.alpha * torch.exp(x) + (x >= 0).type(torch.float)) return grad_output * grad @@ -68,6 +71,7 @@ def initialise(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.xavier_uniform(tensor, gain=nn.init.calculate_gain('selu')) @staticmethod @@ -80,6 +84,7 @@ def initialise_(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.xavier_uniform_(tensor, gain=nn.init.calculate_gain('selu')) @@ -97,4 +102,5 @@ def __init__(self, alpha: float = 1.0507): Args: alpha (float): the alpha parameter. """ + super(ELU, self).__init__(alpha=alpha, scale_factor=1.) diff --git a/analogvnn/nn/activation/Gaussian.py b/analogvnn/nn/activation/Gaussian.py index 8e82c6a..2d0661e 100644 --- a/analogvnn/nn/activation/Gaussian.py +++ b/analogvnn/nn/activation/Gaussian.py @@ -22,6 +22,7 @@ def forward(x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return torch.exp(-torch.pow(x, 2)) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @@ -33,6 +34,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor. """ + x = self.inputs grad = -2 * x * torch.exp(-torch.pow(x, 2)) return grad_output * grad @@ -51,6 +53,7 @@ def forward(x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return (1 / 2) * x * (1 + torch.erf(x / math.sqrt(2))) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @@ -62,6 +65,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor. """ + x = self.inputs grad = (1 / 2) * ( (1 + torch.erf(x / math.sqrt(2))) + x * ((2 / math.sqrt(math.pi)) * torch.exp(-torch.pow(x, 2))) diff --git a/analogvnn/nn/activation/Identity.py b/analogvnn/nn/activation/Identity.py index d687226..136741d 100644 --- a/analogvnn/nn/activation/Identity.py +++ b/analogvnn/nn/activation/Identity.py @@ -22,6 +22,7 @@ def __init__(self, name=None): Args: name (str): the name of the activation function. """ + super(Identity, self).__init__() self.name = name @@ -31,6 +32,7 @@ def extra_repr(self) -> str: Returns: str: the extra representation of the identity activation function. """ + if self.name is not None: return f'name={self.name}' else: @@ -46,6 +48,7 @@ def forward(x: Tensor) -> Tensor: Returns: Tensor: the output tensor same as the input tensor. """ + return x def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @@ -57,4 +60,5 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor same as the gradient of the output tensor. """ + return grad_output diff --git a/analogvnn/nn/activation/ReLU.py b/analogvnn/nn/activation/ReLU.py index f00d576..c7358f9 100644 --- a/analogvnn/nn/activation/ReLU.py +++ b/analogvnn/nn/activation/ReLU.py @@ -27,6 +27,7 @@ def __init__(self, alpha: float): Args: alpha (float): the slope of the negative part of the activation function. """ + super(PReLU, self).__init__() self.alpha = nn.Parameter(torch.tensor(alpha), requires_grad=False) self._zero = nn.Parameter(torch.tensor(0), requires_grad=False) @@ -40,6 +41,7 @@ def forward(self, x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return torch.minimum(self._zero, x) * self.alpha + torch.maximum(self._zero, x) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @@ -51,6 +53,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor. """ + x = self.inputs grad = (x < 0).type(torch.float) * self.alpha + (x >= 0).type(torch.float) return grad_output * grad @@ -65,6 +68,7 @@ def initialise(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.kaiming_uniform(tensor, a=math.sqrt(5), nonlinearity='leaky_relu') @staticmethod @@ -77,6 +81,7 @@ def initialise_(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.kaiming_uniform_(tensor, a=math.sqrt(5), nonlinearity='leaky_relu') @@ -89,6 +94,7 @@ class ReLU(PReLU): def __init__(self): """Initialize the rectified linear unit (ReLU) activation function.""" + super(ReLU, self).__init__(alpha=0) @staticmethod @@ -101,6 +107,7 @@ def initialise(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.kaiming_uniform(tensor, a=math.sqrt(5), nonlinearity='relu') @staticmethod @@ -113,6 +120,7 @@ def initialise_(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.kaiming_uniform_(tensor, a=math.sqrt(5), nonlinearity='relu') @@ -125,4 +133,5 @@ class LeakyReLU(PReLU): def __init__(self): """Initialize the leaky rectified linear unit (LeakyReLU) activation function.""" + super(LeakyReLU, self).__init__(alpha=0.01) diff --git a/analogvnn/nn/activation/SiLU.py b/analogvnn/nn/activation/SiLU.py index afb0265..974f8e1 100644 --- a/analogvnn/nn/activation/SiLU.py +++ b/analogvnn/nn/activation/SiLU.py @@ -21,6 +21,7 @@ def forward(x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return x / (1 + torch.exp(-x)) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @@ -32,6 +33,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor. """ + x = self.inputs neg_e = torch.exp(-x) grad = (1 + neg_e + x * neg_e) / torch.pow(1 + neg_e, 2) diff --git a/analogvnn/nn/activation/Sigmoid.py b/analogvnn/nn/activation/Sigmoid.py index a14e289..2a204ca 100644 --- a/analogvnn/nn/activation/Sigmoid.py +++ b/analogvnn/nn/activation/Sigmoid.py @@ -21,6 +21,7 @@ def forward(x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return 1 / (1 + torch.exp(-x)) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @@ -32,6 +33,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor. """ + x = self.inputs grad = self.forward(x) * (1 - self.forward(x)) return grad_output * grad @@ -46,6 +48,7 @@ def initialise(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.xavier_uniform(tensor, gain=nn.init.calculate_gain('sigmoid')) @staticmethod @@ -58,6 +61,7 @@ def initialise_(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.xavier_uniform_(tensor, gain=nn.init.calculate_gain('sigmoid')) diff --git a/analogvnn/nn/activation/Tanh.py b/analogvnn/nn/activation/Tanh.py index 4d7d61e..4f435a7 100644 --- a/analogvnn/nn/activation/Tanh.py +++ b/analogvnn/nn/activation/Tanh.py @@ -21,6 +21,7 @@ def forward(x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return torch.tanh(x) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @@ -32,6 +33,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor. """ + x = self.inputs grad = 1 - torch.pow(torch.tanh(x), 2) return grad_output * grad @@ -46,6 +48,7 @@ def initialise(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.xavier_uniform(tensor, gain=nn.init.calculate_gain('tanh')) @staticmethod @@ -58,4 +61,5 @@ def initialise_(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ + return nn.init.xavier_uniform_(tensor, gain=nn.init.calculate_gain('tanh')) diff --git a/analogvnn/nn/module/FullSequential.py b/analogvnn/nn/module/FullSequential.py index e31c795..369152d 100644 --- a/analogvnn/nn/module/FullSequential.py +++ b/analogvnn/nn/module/FullSequential.py @@ -22,6 +22,7 @@ def compile(self, device: Optional[torch.device] = None, layer_data: bool = True Returns: FullSequential: self """ + arr = [self.graphs.INPUT, *list(self.registered_modules()), self.graphs.OUTPUT] self.graphs.backward_graph.add_connection(*reversed(arr)) return super().compile(device, layer_data) diff --git a/analogvnn/nn/module/Layer.py b/analogvnn/nn/module/Layer.py index 846144f..5ba6f39 100644 --- a/analogvnn/nn/module/Layer.py +++ b/analogvnn/nn/module/Layer.py @@ -26,6 +26,7 @@ def __nn_Module_init_updated__(function: Callable) -> Callable: Returns: Callable: Wrapped function """ + # noinspection PyUnusedLocal def _temp(*args, **kwargs) -> ...: pass @@ -92,6 +93,7 @@ def __call__(self, *inputs, **kwargs): *inputs: Inputs of the forward pass. **kwargs: Keyword arguments of the forward pass. """ + self._forward_wrapper(self.forward) outputs = super(Layer, self).__call__(*inputs, **kwargs) if self.training: @@ -107,6 +109,7 @@ def use_autograd_graph(self) -> bool: Returns: bool: use_autograd_graph. """ + if self.graphs is not None: return self.graphs.use_autograd_graph return self._use_autograd_graph @@ -118,6 +121,7 @@ def use_autograd_graph(self, use_autograd_graph: bool): Args: use_autograd_graph (bool): use_autograd_graph. """ + self._use_autograd_graph = use_autograd_graph if self.graphs is not None: self.graphs.use_autograd_graph = use_autograd_graph @@ -129,6 +133,7 @@ def inputs(self) -> ArgsKwargsOutput: Returns: ArgsKwargsOutput: inputs. """ + return ArgsKwargs.from_args_kwargs_object(self._inputs) @property @@ -138,6 +143,7 @@ def outputs(self) -> Union[None, Tensor, Sequence[Tensor]]: Returns: Union[None, Tensor, Sequence[Tensor]]: outputs. """ + return self._outputs @property @@ -147,6 +153,7 @@ def backward_function(self) -> Union[None, Callable, BackwardModule]: Returns: Union[None, Callable, BackwardModule]: backward_function. """ + if self._backward_module is not None: return self._backward_module @@ -162,6 +169,7 @@ def backward_function(self, function: Union[BackwardModule, Type[BackwardModule] Args: function (Union[BackwardModule, Type[BackwardModule], Callable]): backward_function. """ + self.set_backward_function(function) def set_backward_function(self, backward_class: Union[Callable, BackwardModule, Type[BackwardModule]]) -> Layer: @@ -176,6 +184,7 @@ def set_backward_function(self, backward_class: Union[Callable, BackwardModule, Raises: TypeError: If backward_class is not a callable or BackwardModule. """ + if backward_class == self: return self @@ -197,8 +206,7 @@ def named_registered_modules( prefix: str = '', remove_duplicate: bool = True ) -> Iterator[Tuple[str, nn.Module]]: - """Returns an iterator over all registered modules in the network, yielding - both the name of the module and the module itself. + """Returns an iterator over registered modules under self. Args: memo: a memo to store the set of modules already added to the result @@ -213,6 +221,7 @@ def named_registered_modules( Duplicate modules are returned only once. In the following example, ``l`` will be returned only once. """ + if memo is None: memo = set() @@ -220,9 +229,9 @@ def named_registered_modules( memo.add(self.backward_function) for name, module in super(Layer, self).named_modules( - memo=memo, - prefix=prefix, - remove_duplicate=remove_duplicate + memo=memo, + prefix=prefix, + remove_duplicate=remove_duplicate ): if module is self: continue @@ -239,6 +248,7 @@ def registered_modules(self) -> Iterator[nn.Module]: Duplicate modules are returned only once. In the following example, ``l`` will be returned only once. """ + for _, module in self.named_registered_modules(): yield module @@ -251,6 +261,7 @@ def _forward_wrapper(self, function: Callable) -> Callable: Returns: Callable: Wrapped function. """ + # noinspection PyUnresolvedReferences if hasattr(function, '__wrapper__') and function.__wrapper__ == Layer._forward_wrapper: return function @@ -284,6 +295,7 @@ def _call_impl_forward(self, *args: Tensor, **kwargs: Tensor) -> TENSORS: Returns: TENSORS: Outputs of the forward pass. """ + if isinstance(self.backward_function, BackwardModule) and self.backward_function.has_forward(): forward_functions = self.backward_function.forward else: diff --git a/analogvnn/nn/module/Model.py b/analogvnn/nn/module/Model.py index ca5e1a2..60ed6e1 100644 --- a/analogvnn/nn/module/Model.py +++ b/analogvnn/nn/module/Model.py @@ -59,6 +59,7 @@ def __init__(self, tensorboard_log_dir=None, device=is_cpu_cuda.device): tensorboard_log_dir (str): The log directory of the tensorboard logger. device (torch.device): The device to run the model on. """ + super(Model, self).__init__() self._compiled = False @@ -89,6 +90,7 @@ def __call__(self, *args, **kwargs): Raises: RuntimeError: if the model is not compiled. """ + if not self._compiled: raise RuntimeError('Model is not compiled yet.') @@ -101,6 +103,7 @@ def use_autograd_graph(self): Returns: bool: If True, the autograd graph is used to calculate the gradients. """ + return self.graphs.use_autograd_graph @use_autograd_graph.setter @@ -110,6 +113,7 @@ def use_autograd_graph(self, use_autograd_graph: bool): Args: use_autograd_graph (bool): If True, the autograd graph is used to calculate the gradients. """ + self.graphs.use_autograd_graph = use_autograd_graph def named_registered_modules( @@ -118,8 +122,7 @@ def named_registered_modules( prefix: str = '', remove_duplicate: bool = True ) -> Iterator[Tuple[str, nn.Module]]: - """Returns an iterator over registered modules under self, - yielding both the name of the module and the module itself. + """Returns an iterator over registered modules under self. Args: memo: a memo to store the set of modules already added to the result @@ -134,6 +137,7 @@ def named_registered_modules( Duplicate modules are returned only once. In the following example, ``l`` will be returned only once. """ + if memo is None: memo = set() @@ -152,6 +156,7 @@ def compile(self, device: Optional[torch.device] = None, layer_data: bool = True Returns: Model: The compiled model. """ + if device is not None: self.device = device @@ -176,6 +181,7 @@ def forward(self, *inputs: Tensor) -> TENSORS: Returns: TENSORS: The output of the model. """ + return self.graphs.forward_graph(inputs, self.training) @torch.no_grad() @@ -188,6 +194,7 @@ def backward(self, *inputs: Tensor) -> TENSORS: Returns: TENSORS: The output of the model. """ + return self.graphs.backward_graph(inputs) def loss(self, output: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]: @@ -203,6 +210,7 @@ def loss(self, output: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]: Raises: ValueError: if loss_function is None. """ + if self.loss_function is None: raise ValueError('loss_function is None') @@ -231,6 +239,7 @@ def train_on(self, train_loader: DataLoader, epoch: int = None, *args, **kwargs) Raises: RuntimeError: if model is not compiled. """ + if self._compiled is False: raise RuntimeError('Model is not compiled') @@ -257,6 +266,7 @@ def test_on(self, test_loader: DataLoader, epoch: int = None, *args, **kwargs) - Raises: RuntimeError: if model is not compiled. """ + if self._compiled is False: raise RuntimeError('Model is not compiled') @@ -285,6 +295,7 @@ def fit( Tuple[float, float, float, float]: The train loss, the train accuracy, the test loss and the test accuracy of the model. """ + train_loss, train_accuracy = self.train_on(train_loader=train_loader, epoch=epoch) test_loss, test_accuracy = self.test_on(test_loader=test_loader, epoch=epoch) return train_loss, train_accuracy, test_loss, test_accuracy @@ -298,6 +309,7 @@ def create_tensorboard(self, log_dir: str) -> TensorboardModelLog: Raises: ImportError: if tensorboard (https://www.tensorflow.org/) is not installed. """ + try: from analogvnn.utils.TensorboardModelLog import TensorboardModelLog except ImportError as e: @@ -316,6 +328,7 @@ def subscribe_tensorboard(self, tensorboard: TensorboardModelLog): Returns: Model: self. """ + self.tensorboard = tensorboard if self._compiled is True: self.tensorboard.on_compile() diff --git a/analogvnn/nn/module/Sequential.py b/analogvnn/nn/module/Sequential.py index ca3d5b4..3fd6dd2 100644 --- a/analogvnn/nn/module/Sequential.py +++ b/analogvnn/nn/module/Sequential.py @@ -25,6 +25,7 @@ def __call__(self, *args, **kwargs): Returns: torch.Tensor: The output of the model. """ + if not self._compiled: self.compile() @@ -40,6 +41,7 @@ def compile(self, device: Optional[torch.device] = None, layer_data: bool = True Returns: Sequential: self """ + arr = [self.graphs.INPUT, *list(self.registered_modules()), self.graphs.OUTPUT] self.graphs.forward_graph.add_connection(*arr) return super().compile(device, layer_data) @@ -50,4 +52,5 @@ def add_sequence(self, *args): Args: *args (nn.Module): The modules to add. """ + return self.extend(args) diff --git a/analogvnn/nn/noise/GaussianNoise.py b/analogvnn/nn/noise/GaussianNoise.py index 4333332..4fe31bd 100644 --- a/analogvnn/nn/noise/GaussianNoise.py +++ b/analogvnn/nn/noise/GaussianNoise.py @@ -41,6 +41,7 @@ def __init__( leakage (float): the leakage of the Gaussian noise. precision (int): the precision of the Gaussian noise. """ + super(GaussianNoise, self).__init__() if (std is None) + (leakage is None) + (precision is None) != 1: @@ -70,6 +71,7 @@ def calc_std(leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_OPE Returns: float: the standard deviation of the Gaussian noise. """ + return 1 / (2 * precision * scipy.special.erfinv(1 - leakage) * math.sqrt(2)) @staticmethod @@ -83,6 +85,7 @@ def calc_precision(std: TENSOR_OPERABLE, leakage: TENSOR_OPERABLE) -> TENSOR_OPE Returns: int: the precision of the Gaussian noise. """ + return 1 / (2 * std * scipy.special.erfinv(1 - leakage) * math.sqrt(2)) @staticmethod @@ -96,6 +99,7 @@ def calc_leakage(std: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_OPE Returns: float: the leakage of the Gaussian noise. """ + return 2 * GaussianNoise.static_cdf(x=-1 / (2 * precision), std=std) @property @@ -105,6 +109,7 @@ def stddev(self) -> Tensor: Returns: Tensor: the standard deviation of the Gaussian noise. """ + return self.std @property @@ -114,6 +119,7 @@ def variance(self) -> Tensor: Returns: Tensor: the variance of the Gaussian noise. """ + return self.stddev.pow(2) def pdf(self, x: Tensor, mean: Tensor = 0) -> Tensor: @@ -126,6 +132,7 @@ def pdf(self, x: Tensor, mean: Tensor = 0) -> Tensor: Returns: Tensor: the probability density function of the Gaussian noise. """ + return torch.exp(self.log_prob(x=x, mean=mean)) def log_prob(self, x: Tensor, mean: Tensor = 0) -> Tensor: @@ -138,6 +145,7 @@ def log_prob(self, x: Tensor, mean: Tensor = 0) -> Tensor: Returns: Tensor: the log probability density function of the Gaussian noise. """ + x = x if isinstance(x, Tensor) else torch.tensor(x, requires_grad=False) mean = mean if isinstance(mean, Tensor) else torch.tensor(mean, requires_grad=False) @@ -157,6 +165,7 @@ def static_cdf(x: TENSOR_OPERABLE, std: TENSOR_OPERABLE, mean: TENSOR_OPERABLE = Returns: TENSOR_OPERABLE: the cumulative distribution function of the Gaussian noise. """ + return 1 / 2 * (1 + math.erf((x - mean) / (std * math.sqrt(2)))) def cdf(self, x: Tensor, mean: Tensor = 0) -> Tensor: @@ -169,6 +178,7 @@ def cdf(self, x: Tensor, mean: Tensor = 0) -> Tensor: Returns: Tensor: the cumulative distribution function of the Gaussian noise. """ + x = x if isinstance(x, Tensor) else torch.tensor(x, requires_grad=False) mean = mean if isinstance(mean, Tensor) else torch.tensor(mean, requires_grad=False) return self.static_cdf(x=x, std=self.std, mean=mean) @@ -182,6 +192,7 @@ def forward(self, x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return torch.normal(mean=x, std=self.std) def extra_repr(self) -> str: @@ -190,4 +201,5 @@ def extra_repr(self) -> str: Returns: str: the extra representation of the Gaussian noise. """ + return f'std={float(self.std):.4f}, leakage={float(self.leakage):.4f}, precision={int(self.precision)}' diff --git a/analogvnn/nn/noise/LaplacianNoise.py b/analogvnn/nn/noise/LaplacianNoise.py index b88bc2f..ab2b00e 100644 --- a/analogvnn/nn/noise/LaplacianNoise.py +++ b/analogvnn/nn/noise/LaplacianNoise.py @@ -40,6 +40,7 @@ def __init__( leakage (float): the leakage of the Laplacian noise. precision (int): the precision of the Laplacian noise. """ + super(LaplacianNoise, self).__init__() if (scale is None) + (leakage is None) + (precision is None) != 1: @@ -69,6 +70,7 @@ def calc_scale(leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_O Returns: float: the scale of the Laplacian noise. """ + return - 1 / (2 * math.log(leakage) * precision) @staticmethod @@ -82,6 +84,7 @@ def calc_precision(scale: TENSOR_OPERABLE, leakage: TENSOR_OPERABLE) -> TENSOR_O Returns: int: the precision of the Laplacian noise. """ + return - 1 / (2 * math.log(leakage) * scale) @staticmethod @@ -95,6 +98,7 @@ def calc_leakage(scale: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> Tensor: Returns: float: the leakage of the Laplacian noise. """ + return 2 * LaplacianNoise.static_cdf(x=-1 / (2 * precision), scale=scale) @property @@ -104,6 +108,7 @@ def stddev(self) -> Tensor: Returns: Tensor: the standard deviation of the Laplacian noise. """ + return (2 ** 0.5) * self.scale @property @@ -113,6 +118,7 @@ def variance(self) -> Tensor: Returns: Tensor: the variance of the Laplacian noise. """ + return 2 * self.scale.pow(2) def pdf(self, x: TENSOR_OPERABLE, loc: TENSOR_OPERABLE = 0) -> Tensor: @@ -125,6 +131,7 @@ def pdf(self, x: TENSOR_OPERABLE, loc: TENSOR_OPERABLE = 0) -> Tensor: Returns: Tensor: the probability density function of the Laplacian noise. """ + return torch.exp(self.log_prob(x=x, loc=loc)) def log_prob(self, x: TENSOR_OPERABLE, loc: TENSOR_OPERABLE = 0) -> Tensor: @@ -137,6 +144,7 @@ def log_prob(self, x: TENSOR_OPERABLE, loc: TENSOR_OPERABLE = 0) -> Tensor: Returns: Tensor: the log probability density function of the Laplacian noise. """ + x = x if isinstance(x, Tensor) else torch.tensor(x, requires_grad=False) loc = loc if isinstance(loc, Tensor) else torch.tensor(loc, requires_grad=False) return -torch.log(2 * self.scale) - torch.abs(x - loc) / self.scale @@ -153,6 +161,7 @@ def static_cdf(x: TENSOR_OPERABLE, scale: TENSOR_OPERABLE, loc: TENSOR_OPERABLE Returns: TENSOR_OPERABLE: the cumulative distribution function of the Laplacian noise. """ + return 0.5 - 0.5 * np.sign(x - loc) * np.expm1(-abs(x - loc) / scale) def cdf(self, x: Tensor, loc: Tensor = 0) -> Tensor: @@ -165,6 +174,7 @@ def cdf(self, x: Tensor, loc: Tensor = 0) -> Tensor: Returns: Tensor: the cumulative distribution function of the Laplacian noise. """ + x = x if isinstance(x, Tensor) else torch.tensor(x, requires_grad=False) loc = loc if isinstance(loc, Tensor) else torch.tensor(loc, requires_grad=False) return self.static_cdf(x=x, scale=self.scale, loc=loc) @@ -178,6 +188,7 @@ def forward(self, x: Tensor) -> Tensor: Returns: Tensor: the output tensor with Laplacian noise. """ + return torch.distributions.Laplace(loc=x, scale=self.scale).sample() def extra_repr(self) -> str: @@ -186,4 +197,5 @@ def extra_repr(self) -> str: Returns: str: the extra representation of the Laplacian noise. """ + return f'std={float(self.std):.4f}, leakage={float(self.leakage):.4f}, precision={int(self.precision)}' diff --git a/analogvnn/nn/noise/PoissonNoise.py b/analogvnn/nn/noise/PoissonNoise.py index df05371..39a14ad 100644 --- a/analogvnn/nn/noise/PoissonNoise.py +++ b/analogvnn/nn/noise/PoissonNoise.py @@ -42,6 +42,7 @@ def __init__( max_leakage (Optional[float]): the maximum leakage of the Poisson noise. precision (Optional[int]): the precision of the Poisson noise. """ + super(PoissonNoise, self).__init__() if (scale is None) + (max_leakage is None) + (precision is None) != 1: @@ -75,6 +76,7 @@ def calc_scale(max_leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE, max_che Returns: TENSOR_OPERABLE: the scale of the Poisson noise function. """ + max_leakage = float(max_leakage) precision = float(precision) r, _ = toms748( @@ -97,6 +99,7 @@ def calc_precision(scale: TENSOR_OPERABLE, max_leakage: TENSOR_OPERABLE, max_che Returns: TENSOR_OPERABLE: the precision of the Poisson noise. """ + max_leakage = float(max_leakage) scale = float(scale) r, _ = toms748( @@ -118,6 +121,7 @@ def calc_max_leakage(scale: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENS Returns: TENSOR_OPERABLE: the maximum leakage of the Poisson noise. """ + return 1 - ( PoissonNoise.static_cdf(x=1. + 1 / (2 * precision), rate=1., scale_factor=scale * precision) - PoissonNoise.static_cdf(x=1. - 1 / (2 * precision), rate=1., scale_factor=scale * precision) @@ -135,6 +139,7 @@ def static_cdf(x: TENSOR_OPERABLE, rate: TENSOR_OPERABLE, scale_factor: TENSOR_O Returns: TENSOR_OPERABLE: the cumulative distribution function of the Poisson noise. """ + if np.isclose(rate, np.zeros_like(rate)): return np.ones_like(x) @@ -153,6 +158,7 @@ def staticmethod_leakage(scale: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> Returns: TENSOR_OPERABLE: the leakage of the Poisson noise. """ + cdf_domain = np.linspace(-1, 1, int(2 * precision + 1), dtype=float) correctness = 0 @@ -185,6 +191,7 @@ def leakage(self) -> float: Returns: float: the leakage of the Poisson noise. """ + return self.staticmethod_leakage(scale=float(self.scale), precision=int(self.precision)) @property @@ -194,6 +201,7 @@ def rate_factor(self) -> Tensor: Returns: Tensor: the rate factor of the Poisson noise. """ + return self.precision * self.scale def pdf(self, x: Tensor, rate: Tensor) -> Tensor: @@ -206,6 +214,7 @@ def pdf(self, x: Tensor, rate: Tensor) -> Tensor: Returns: Tensor: the probability density function of the Poisson noise. """ + x = x if isinstance(x, Tensor) else torch.tensor(x, requires_grad=False) rate = rate if isinstance(rate, Tensor) else torch.tensor(rate, requires_grad=False) @@ -224,6 +233,7 @@ def log_prob(self, x: Tensor, rate: Tensor) -> Tensor: Returns: Tensor: the log probability of the Poisson noise. """ + x = x if isinstance(x, Tensor) else torch.tensor(x, requires_grad=False) rate = rate if isinstance(rate, Tensor) else torch.tensor(rate, requires_grad=False) @@ -241,6 +251,7 @@ def cdf(self, x: Tensor, rate: Tensor) -> Tensor: Returns: Tensor: the cumulative distribution function of the Poisson noise. """ + x = x if isinstance(x, Tensor) else torch.tensor(x, requires_grad=False) rate = rate if isinstance(rate, Tensor) else torch.tensor(rate, requires_grad=False) return self.static_cdf(x, rate, self.rate_factor) @@ -254,6 +265,7 @@ def forward(self, x: Tensor) -> Tensor: Returns: Tensor: the output of the Poisson noise. """ + return torch.sign(x) * torch.poisson(torch.abs(x * self.rate_factor)) / self.rate_factor def extra_repr(self) -> str: @@ -262,6 +274,7 @@ def extra_repr(self) -> str: Returns: str: the extra representation of the Poisson noise. """ + return f'scale={float(self.scale):.4f}' \ f', max_leakage={float(self.max_leakage):.4f}' \ f', leakage={float(self.leakage):.4f}' \ diff --git a/analogvnn/nn/noise/UniformNoise.py b/analogvnn/nn/noise/UniformNoise.py index ac3a49d..7e73652 100644 --- a/analogvnn/nn/noise/UniformNoise.py +++ b/analogvnn/nn/noise/UniformNoise.py @@ -42,6 +42,7 @@ def __init__( leakage (float): the leakage of the uniform noise. precision (int): the precision of the uniform noise. """ + super(UniformNoise, self).__init__() if (low is None or high is None) + (leakage is None) + (precision is None) != 1: @@ -71,6 +72,7 @@ def calc_high_low(leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> Tuple Returns: Tuple[TENSOR_OPERABLE, TENSOR_OPERABLE]: the high and low of the uniform noise. """ + v = 1 / (1 - leakage) * (1 / precision) return -v / 2, v / 2 @@ -86,6 +88,7 @@ def calc_leakage(low: TENSOR_OPERABLE, high: TENSOR_OPERABLE, precision: TENSOR_ Returns: TENSOR_OPERABLE: the leakage of the uniform noise. """ + return 1 - min(1, (1 / precision) * (1 / (high - low))) @staticmethod @@ -100,6 +103,7 @@ def calc_precision(low: TENSOR_OPERABLE, high: TENSOR_OPERABLE, leakage: TENSOR_ Returns: TENSOR_OPERABLE: the precision of the uniform noise. """ + return 1 / (1 - leakage) * (1 / (high - low)) @property @@ -109,6 +113,7 @@ def mean(self) -> Tensor: Returns: Tensor: the mean of the uniform noise. """ + return (self.high + self.low) / 2 @property @@ -118,6 +123,7 @@ def stddev(self) -> Tensor: Returns: Tensor: the standard deviation of the uniform noise. """ + return (self.high - self.low) / 12 ** 0.5 @property @@ -127,6 +133,7 @@ def variance(self) -> Tensor: Returns: Tensor: the variance of the uniform noise. """ + return (self.high - self.low).pow(2) / 12 def pdf(self, x: Tensor) -> Tensor: @@ -138,6 +145,7 @@ def pdf(self, x: Tensor) -> Tensor: Returns: Tensor: the probability density function of the uniform noise. """ + return torch.exp(self.log_prob(x=x)) def log_prob(self, x: Tensor) -> Tensor: @@ -149,6 +157,7 @@ def log_prob(self, x: Tensor) -> Tensor: Returns: Tensor: the log probability density function of the uniform noise. """ + lb = self.low.le(x).type_as(self.low) ub = self.high.gt(x).type_as(self.low) return torch.log(lb.mul(ub)) - torch.log(self.high - self.low) @@ -162,6 +171,7 @@ def cdf(self, x: TENSOR_OPERABLE) -> TENSOR_OPERABLE: Returns: TENSOR_OPERABLE: the cumulative distribution function of the uniform noise. """ + result = (x - self.low) / (self.high - self.low) return result.clamp(min=0, max=1) @@ -174,6 +184,7 @@ def forward(self, x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return torch.distributions.Uniform(low=x + self.low, high=x + self.high).sample() def extra_repr(self) -> str: @@ -182,6 +193,7 @@ def extra_repr(self) -> str: Returns: str: the extra representation of the uniform noise. """ + return f'high={float(self.high):.4f}' \ f', low={float(self.low):.4f}' \ f', leakage={float(self.leakage):.4f}' \ diff --git a/analogvnn/nn/normalize/Clamp.py b/analogvnn/nn/normalize/Clamp.py index 7d96e56..7db26fc 100644 --- a/analogvnn/nn/normalize/Clamp.py +++ b/analogvnn/nn/normalize/Clamp.py @@ -21,6 +21,7 @@ def forward(x: Tensor): Returns: Tensor: the output tensor. """ + return torch.clamp(x, min=-1, max=1) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @@ -32,6 +33,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor. """ + x = self.inputs grad = ((-1 <= x) * (x <= 1.)).type(torch.float) return grad_output * grad @@ -50,6 +52,7 @@ def forward(x: Tensor): Returns: Tensor: the output tensor. """ + return torch.clamp(x, min=0, max=1) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @@ -61,6 +64,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: Returns: Optional[Tensor]: the gradient of the input tensor. """ + x = self.inputs grad = ((0 <= x) * (x <= 1.)).type(torch.float) return grad_output * grad diff --git a/analogvnn/nn/normalize/LPNorm.py b/analogvnn/nn/normalize/LPNorm.py index 27bcce0..68cc2d5 100644 --- a/analogvnn/nn/normalize/LPNorm.py +++ b/analogvnn/nn/normalize/LPNorm.py @@ -25,6 +25,7 @@ def __init__(self, p: int, make_max_1=False): p (int): the pth power of the Lp norm. make_max_1 (bool): if True, the maximum absolute value of the output tensor will be 1. """ + super(LPNorm, self).__init__() self.p = nn.Parameter(torch.tensor(p), requires_grad=False) self.make_max_1 = nn.Parameter(torch.tensor(make_max_1), requires_grad=False) @@ -38,6 +39,7 @@ def forward(self, x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + norm = x if len(x.shape) > 1: norm = torch.flatten(norm, start_dim=1) @@ -64,6 +66,7 @@ def forward(self, x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + norm = torch.norm(x, self.p) norm = torch.clamp(norm, min=1e-4) x = torch.div(x, norm) @@ -79,6 +82,7 @@ class L1Norm(LPNorm): def __init__(self): """Initializes the row-wise L1 normalization function.""" + super(L1Norm, self).__init__(p=1, make_max_1=False) @@ -87,6 +91,7 @@ class L2Norm(LPNorm): def __init__(self): """Initializes the row-wise L2 normalization function.""" + super(L2Norm, self).__init__(p=2, make_max_1=False) @@ -95,6 +100,7 @@ class L1NormW(LPNormW): def __init__(self): """Initializes the whole matrix L1 normalization function.""" + super(L1NormW, self).__init__(p=1, make_max_1=False) @@ -103,6 +109,7 @@ class L2NormW(LPNormW): def __init__(self): """Initializes the whole matrix L2 normalization function.""" + super(L2NormW, self).__init__(p=2, make_max_1=False) @@ -111,6 +118,7 @@ class L1NormM(LPNorm): def __init__(self): """Initializes the row-wise L1 normalization function with maximum absolute value of 1.""" + super(L1NormM, self).__init__(p=1, make_max_1=True) @@ -119,6 +127,7 @@ class L2NormM(LPNorm): def __init__(self): """Initializes the row-wise L2 normalization function with maximum absolute value of 1.""" + super(L2NormM, self).__init__(p=2, make_max_1=True) @@ -127,6 +136,7 @@ class L1NormWM(LPNormW): def __init__(self): """Initializes the whole matrix L1 normalization function with maximum absolute value of 1.""" + super(L1NormWM, self).__init__(p=1, make_max_1=True) @@ -135,4 +145,5 @@ class L2NormWM(LPNormW): def __init__(self): """Initializes the whole matrix L2 normalization function with maximum absolute value of 1.""" + super(L2NormWM, self).__init__(p=2, make_max_1=True) diff --git a/analogvnn/nn/precision/ReducePrecision.py b/analogvnn/nn/precision/ReducePrecision.py index 3b5270e..207889a 100644 --- a/analogvnn/nn/precision/ReducePrecision.py +++ b/analogvnn/nn/precision/ReducePrecision.py @@ -30,6 +30,7 @@ def __init__(self, precision: int = None, divide: float = 0.5): divide (float): the rounding value that is if divide is 0.5, then 0.6 will be rounded to 1.0 and 0.4 will be rounded to 0.0. """ + super(ReducePrecision, self).__init__() if precision < 1: raise ValueError(f'precision has to be more than 0, but got {precision}') @@ -50,6 +51,7 @@ def precision_width(self) -> Tensor: Returns: Tensor: the precision width """ + return 1 / self.precision @property @@ -59,6 +61,7 @@ def bit_precision(self) -> Tensor: Returns: Tensor: the bit precision of the ReducePrecision module. """ + return torch.log2(self.precision + 1) @staticmethod @@ -71,6 +74,7 @@ def convert_to_precision(bit_precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: Returns: TENSOR_OPERABLE: the precision. """ + return 2 ** bit_precision - 1 def extra_repr(self) -> str: @@ -79,6 +83,7 @@ def extra_repr(self) -> str: Returns: str: string """ + return f'precision={int(self.precision)}, divide={float(self.divide):0.2f}' def forward(self, x: Tensor) -> Tensor: @@ -90,4 +95,5 @@ def forward(self, x: Tensor) -> Tensor: Returns: Tensor: the output tensor. """ + return reduce_precision(x, self.precision, self.divide) diff --git a/analogvnn/nn/precision/StochasticReducePrecision.py b/analogvnn/nn/precision/StochasticReducePrecision.py index 7e36ae9..dad532a 100644 --- a/analogvnn/nn/precision/StochasticReducePrecision.py +++ b/analogvnn/nn/precision/StochasticReducePrecision.py @@ -25,6 +25,7 @@ def __init__(self, precision: int = 8): Args: precision (int): the precision of the output tensor. """ + super(StochasticReducePrecision, self).__init__() if precision < 1: raise ValueError('precision has to be more than 0, but got {}'.format(precision)) @@ -41,6 +42,7 @@ def precision_width(self) -> Tensor: Returns: Tensor: the precision width """ + return 1 / self.precision @property @@ -50,6 +52,7 @@ def bit_precision(self) -> Tensor: Returns: Tensor: the bit precision of the ReducePrecision module. """ + return torch.log2(self.precision + 1) @staticmethod @@ -62,6 +65,7 @@ def convert_to_precision(bit_precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: Returns: TENSOR_OPERABLE: the precision. """ + return 2 ** bit_precision - 1 def extra_repr(self) -> str: @@ -70,6 +74,7 @@ def extra_repr(self) -> str: Returns: str: string """ + return f'precision={self.precision}' def forward(self, x: Tensor) -> Tensor: @@ -81,4 +86,5 @@ def forward(self, x: Tensor) -> Tensor: Returns: Tensor: output tensor. """ + return stochastic_reduce_precision(x, self.precision) diff --git a/analogvnn/parameter/Parameter.py b/analogvnn/parameter/Parameter.py index 54bdb2e..2999efe 100644 --- a/analogvnn/parameter/Parameter.py +++ b/analogvnn/parameter/Parameter.py @@ -21,6 +21,7 @@ def __new__(cls, data=None, requires_grad=True, *args, **kwargs): Returns: Parameter: the created parameter. """ + return super(Parameter, cls).__new__(cls, data, requires_grad) # noinspection PyUnusedLocal @@ -33,6 +34,7 @@ def __init__(self, data=None, requires_grad=True, *args, **kwargs): *args: additional arguments. **kwargs: additional keyword arguments. """ + super(Parameter, self).__init__() def __repr__(self, *args, **kwargs): @@ -45,4 +47,5 @@ def __repr__(self, *args, **kwargs): Returns: str: the string representation. """ + return super(Parameter, self).__repr__(*args, **kwargs) diff --git a/analogvnn/parameter/PseudoParameter.py b/analogvnn/parameter/PseudoParameter.py index 0d7b7bf..5b14945 100644 --- a/analogvnn/parameter/PseudoParameter.py +++ b/analogvnn/parameter/PseudoParameter.py @@ -31,6 +31,7 @@ def __init__(self, original, transformed): original (PseudoParameter): the original parameters. transformed (nn.Parameter): the transformed parameters. """ + super(PseudoParameterModule, self).__init__() self.original = original self._transformed = transformed @@ -45,6 +46,7 @@ def __call__(self, *args, **kwargs) -> nn.Parameter: Returns: nn.Parameter: The transformed parameter. """ + return self.original() forward = __call__ @@ -62,6 +64,7 @@ def set_original_data(self, data: Tensor) -> PseudoParameterModule: Returns: PseudoParameterModule: self. """ + self.original.data = data return self @@ -108,6 +111,7 @@ def identity(x: Any) -> Any: Returns: Any: the input tensor. """ + return x def __init__(self, data=None, requires_grad=True, transformation=None, *args, **kwargs): @@ -120,6 +124,7 @@ def __init__(self, data=None, requires_grad=True, transformation=None, *args, ** *args: additional arguments. **kwargs: additional keyword arguments. """ + super(PseudoParameter, self).__init__(data, requires_grad, *args, **kwargs) self._transformed = nn.Parameter(data=data, requires_grad=requires_grad) self._transformed.original = self @@ -144,6 +149,7 @@ def __call__(self, *args, **kwargs): Raises: RuntimeError: if the transformation callable fails. """ + try: self._transformed.data = self._transformation(self) except Exception as e: @@ -156,6 +162,7 @@ def __repr__(self): Returns: str: the string representation. """ + return f'{PseudoParameter.__name__}(' \ f'transform={self.transformation}' \ f', data={self.data}' \ @@ -168,6 +175,7 @@ def grad(self): Returns: Tensor: the gradient. """ + return self._transformed.grad @property @@ -177,6 +185,7 @@ def module(self): Returns: PseudoParameterModule: the module. """ + return self._module @property @@ -186,6 +195,7 @@ def transformation(self): Returns: Callable: the transformation. """ + return self._transformation @transformation.setter @@ -195,6 +205,7 @@ def transformation(self, transformation: Callable): Args: transformation (Callable): the transformation. """ + self.set_transformation(transformation) def set_transformation(self, transformation) -> PseudoParameter: @@ -206,6 +217,7 @@ def set_transformation(self, transformation) -> PseudoParameter: Returns: PseudoParameter: self. """ + self._transformation = transformation if isinstance(self._transformation, nn.Module): self._transformation.eval() @@ -223,6 +235,7 @@ def parameterize(cls, module: nn.Module, param_name: str, transformation: Callab Returns: PseudoParameter: the parameterized parameter. """ + assert hasattr(module, param_name) param = getattr(module, param_name) @@ -249,6 +262,7 @@ def parametrize_module(cls, module: nn.Module, transformation: Callable, require transformation (Callable): the transformation. requires_grad (bool): if True, only parametrized parameters that require gradients. """ + with torch.no_grad(): for name, parameter in list(module.named_parameters(recurse=False)): if isinstance(parameter, cls): diff --git a/analogvnn/utils/TensorboardModelLog.py b/analogvnn/utils/TensorboardModelLog.py index 13f08e1..5ee65aa 100644 --- a/analogvnn/utils/TensorboardModelLog.py +++ b/analogvnn/utils/TensorboardModelLog.py @@ -38,6 +38,7 @@ def __init__(self, model: Model, log_dir: str): model (nn.Module): the model to log. log_dir (str): the directory to log to. """ + super(TensorboardModelLog, self).__init__() self.model = model self.tensorboard = None @@ -63,6 +64,7 @@ def set_log_dir(self, log_dir: str) -> TensorboardModelLog: Raises: ValueError: if the log directory is invalid. """ + # https://github.com/tensorflow/tensorboard/pull/6135 from tensorboard.compat import tf if getattr(tf, 'io', None) is None: @@ -81,6 +83,7 @@ def _add_layer_data(self, epoch: int = None): Args: epoch (int): the epoch to add the data for. """ + for name, parameter in self.model.named_parameters(): if not parameter.requires_grad: continue @@ -93,6 +96,7 @@ def on_compile(self, layer_data: bool = True): Args: layer_data (bool): whether to log the layer data. """ + if self.layer_data: self.layer_data = layer_data @@ -116,6 +120,7 @@ def add_graph( Returns: TensorboardModelLog: self. """ + if model is None: model = self.model @@ -161,6 +166,7 @@ def add_summary( Raises: ImportError: if torchinfo (https://github.com/tyleryep/torchinfo) is not installed. """ + try: import torchinfo except ImportError as e: @@ -220,6 +226,7 @@ def register_training(self, epoch: int, train_loss: float, train_accuracy: float Returns: TensorboardModelLog: self. """ + self.tensorboard.add_scalar('Loss/train', train_loss, epoch) self.tensorboard.add_scalar('Accuracy/train', train_accuracy, epoch) if self.layer_data: @@ -237,6 +244,7 @@ def register_testing(self, epoch: int, test_loss: float, test_accuracy: float) - Returns: TensorboardModelLog: self. """ + self.tensorboard.add_scalar('Loss/test', test_loss, epoch) self.tensorboard.add_scalar('Accuracy/test', test_accuracy, epoch) return self @@ -249,6 +257,7 @@ def close(self, *args, **kwargs): *args: ignored. **kwargs: ignored. """ + if self.tensorboard is not None: self.tensorboard.close() self.tensorboard = None @@ -259,6 +268,7 @@ def __enter__(self): Returns: TensorboardModelLog: self. """ + return self __exit__ = close diff --git a/analogvnn/utils/is_cpu_cuda.py b/analogvnn/utils/is_cpu_cuda.py index c5771d3..8d7d7c6 100644 --- a/analogvnn/utils/is_cpu_cuda.py +++ b/analogvnn/utils/is_cpu_cuda.py @@ -20,6 +20,7 @@ class CPUCuda: def __init__(self): """Initialize the CPUCuda class.""" + super(CPUCuda, self).__init__() self._device = None self.device_name = None @@ -31,6 +32,7 @@ def use_cpu(self) -> CPUCuda: Returns: CPUCuda: self """ + self.set_device('cpu') return self @@ -40,6 +42,7 @@ def use_cuda_if_available(self) -> CPUCuda: Returns: CPUCuda: self """ + if torch.cuda.is_available(): self.set_device(f'cuda:{torch.cuda.current_device()}') return self @@ -53,6 +56,7 @@ def set_device(self, device_name: str) -> CPUCuda: Returns: CPUCuda: self """ + self._device = torch.device(device_name) self.device_name = self._device.type return self @@ -64,6 +68,7 @@ def device(self) -> torch.device: Returns: torch.device: the device. """ + return self._device @property @@ -73,6 +78,7 @@ def is_cpu(self) -> bool: Returns: bool: True if the device is cpu, False otherwise. """ + return self.device_name.startswith('cpu') @property @@ -82,6 +88,7 @@ def is_cuda(self) -> bool: Returns: bool: True if the device is cuda, False otherwise. """ + return self.device_name.startswith('cuda') @property @@ -91,6 +98,7 @@ def is_using_cuda(self) -> Tuple[torch.device, bool]: Returns: tuple: the device and True if the device is cuda, False otherwise. """ + return self.device, self.is_cuda def get_module_device(self, module) -> torch.device: @@ -102,6 +110,7 @@ def get_module_device(self, module) -> torch.device: Returns: torch.device: the device of the module. """ + # noinspection PyBroadException try: device: torch.device = getattr(module, 'device', None) diff --git a/analogvnn/utils/render_autograd_graph.py b/analogvnn/utils/render_autograd_graph.py index e6f9f65..d2edf16 100644 --- a/analogvnn/utils/render_autograd_graph.py +++ b/analogvnn/utils/render_autograd_graph.py @@ -49,6 +49,7 @@ def size_to_str(size): Returns: str: the string representation of the size. """ + return '(' + ', '.join(['%d' % s for s in size]) + ')' @@ -67,6 +68,7 @@ def resize_graph(dot: Digraph, size_per_element: float = 0.15, min_size: float = size_per_element**2 pixels. min_size (float): Minimum size of graph. """ + # Get the approximate number of nodes and edges num_rows = len(dot.body) content_size = num_rows * size_per_element @@ -86,6 +88,7 @@ def get_fn_name(fn: Callable, show_attrs: bool, max_attr_chars: int) -> str: Returns: str: the name of the function. """ + name = str(type(fn).__name__) if name.endswith('Backward'): name = name[:-8] @@ -159,6 +162,7 @@ def __post_init__(self): Raises: ImportError: if graphviz (https://pygraphviz.github.io/) is not available. """ + try: from graphviz import Digraph except ImportError as e: @@ -184,6 +188,7 @@ def inputs(self) -> Sequence[Tensor]: Returns: Sequence[Tensor]: the arg inputs to the module. """ + return self._inputs @inputs.setter @@ -193,6 +198,7 @@ def inputs(self, inputs: Union[Tensor, Sequence[Tensor]]): Args: inputs (Union[Tensor, Sequence[Tensor]]): the inputs to the module. """ + self._inputs = inputs self._called = False @@ -213,6 +219,7 @@ def inputs_kwargs(self) -> Dict[str, Tensor]: Args: Dict[str, Tensor]: the keyword inputs to the module. """ + return self._inputs_kwargs @inputs_kwargs.setter @@ -222,6 +229,7 @@ def inputs_kwargs(self, inputs_kwargs: Dict[str, Tensor]): Args: inputs_kwargs (Dict[str, Tensor]): the keyword inputs to the module. """ + self._inputs_kwargs = inputs_kwargs self._called = False @@ -239,6 +247,7 @@ def outputs(self) -> Optional[Sequence[Tensor]]: Returns: Optional[Sequence[Tensor]]: the outputs of the module. """ + return self._outputs @outputs.setter @@ -259,6 +268,7 @@ def module(self) -> nn.Module: Returns: nn.Module: the module to be traced. """ + return self._module @module.setter @@ -268,6 +278,7 @@ def module(self, module: nn.Module): Args: module (nn.Module): the module. """ + self._module = module self._called = False @@ -285,6 +296,7 @@ def reset_params(self): Returns: AutoGradDot: self. """ + self.param_map = {} self._seen = set() self.inputs = self.inputs @@ -299,6 +311,7 @@ def ignore_tensor(self) -> Dict[int, bool]: Returns: Dict[int, bool]: the ignore tensor dict. """ + return self._ignore_tensor def add_ignore_tensor(self, tensor: Tensor): @@ -310,6 +323,7 @@ def add_ignore_tensor(self, tensor: Tensor): Returns: AutoGradDot: self. """ + self._ignore_tensor[id(tensor)] = True return self @@ -322,6 +336,7 @@ def del_ignore_tensor(self, tensor: Tensor): Returns: AutoGradDot: self. """ + self._ignore_tensor.pop(id(tensor), None) return self @@ -335,6 +350,7 @@ def get_tensor_name(self, tensor: Tensor, name: Optional[str] = None) -> Tuple[s Returns: Tuple[str, str]: the name and size of the tensor. """ + if not name: if id(tensor) in self.param_map: name = self.param_map[id(tensor)] @@ -362,6 +378,7 @@ def add_tensor(self, tensor: Tensor, name: Optional[str] = None, _attributes=Non Returns: AutoGradDot: self. """ + self._seen.add(tensor) self.dot.node( name=str(id(tensor)), @@ -382,6 +399,7 @@ def add_fn(self, fn: Any, _attributes=None, **kwargs): Returns: AutoGradDot: self. """ + self._seen.add(fn) self.dot.node( name=str(id(fn)), @@ -404,6 +422,7 @@ def add_edge(self, u: Any, v: Any, label: Optional[str] = None, _attributes=None Returns: AutoGradDot: self. """ + self.dot.edge( tail_name=str(id(u)), head_name=str(id(v)), @@ -422,6 +441,7 @@ def add_seen(self, item: Any): Returns: AutoGradDot: self. """ + self._seen.add(item) return self @@ -434,6 +454,7 @@ def is_seen(self, item: Any) -> bool: Returns: bool: True if the item is in the seen set. """ + return item in self._seen @@ -444,6 +465,7 @@ def _add_grad_fn(link: Union[Tensor, Callable], autograd_dot: AutoGradDot) -> Op link (Union[Tensor, Callable]): the Tensor or Tensor.grad_fn. autograd_dot (AutoGradDot): the AutoGradDot object. """ + if autograd_dot.is_seen(link): return None @@ -558,6 +580,7 @@ def _compile_autograd_obj( Returns: AutoGradDot: graphviz representation of autograd graph """ + if LooseVersion(torch.__version__) < LooseVersion('1.9') and (show_attrs or show_saved): warnings.warn( 'make_dot: showing grad_fn attributes and saved variables' @@ -625,6 +648,7 @@ def make_autograd_obj_from_outputs( Returns: AutoGradDot: graphviz representation of autograd graph """ + autograd_dot = AutoGradDot() autograd_dot.outputs = outputs named_params = dict(named_params) @@ -665,6 +689,7 @@ def make_autograd_obj_from_module( Returns: AutoGradDot: graphviz representation of autograd graph """ + assert isinstance(module, nn.Module) new_args = [] new_kwargs = {} @@ -773,6 +798,7 @@ def get_autograd_dot_from_trace(trace) -> Digraph: Returns: graphviz.Digraph: the resulting graph. """ + try: from graphviz import Digraph except ImportError as e: @@ -835,6 +861,7 @@ def get_autograd_dot_from_outputs( Returns: Digraph: graphviz representation of autograd graph """ + return make_autograd_obj_from_outputs( outputs=outputs, named_params=named_params, @@ -874,6 +901,7 @@ def get_autograd_dot_from_module( Returns: Digraph: graphviz representation of autograd graph """ + return make_autograd_obj_from_module( module, *args, @@ -913,6 +941,7 @@ def save_autograd_graph_from_outputs( Returns: str: The (possibly relative) path of the rendered file. """ + return get_autograd_dot_from_outputs( outputs=outputs, named_params=named_params, @@ -954,6 +983,7 @@ def save_autograd_graph_from_module( Returns: str: The (possibly relative) path of the rendered file. """ + return get_autograd_dot_from_module( module, *args, @@ -976,4 +1006,5 @@ def save_autograd_graph_from_trace(filename: Union[str, Path], trace) -> str: Returns: str: The (possibly relative) path of the rendered file. """ + return get_autograd_dot_from_trace(trace).render(filename, format='svg', cleanup=True) diff --git a/analogvnn/utils/to_tensor_parameter.py b/analogvnn/utils/to_tensor_parameter.py index 3650688..7956a22 100644 --- a/analogvnn/utils/to_tensor_parameter.py +++ b/analogvnn/utils/to_tensor_parameter.py @@ -17,6 +17,7 @@ def to_float_tensor(*args) -> Tuple[Union[torch.Tensor, None], ...]: Returns: tuple: the converted arguments. """ + return tuple((None if i is None else torch.tensor(i, requires_grad=False, dtype=torch.float)) for i in args) @@ -31,4 +32,5 @@ def to_nongrad_parameter(*args) -> Tuple[Union[nn.Parameter, None], ...]: Returns: tuple: the converted arguments. """ + return tuple((None if i is None else nn.Parameter(i, requires_grad=False)) for i in to_float_tensor(*args)) diff --git a/docs/_static/analogvnn-logo-square-black.svg b/docs/_static/analogvnn-logo-square-black.svg index 6001b17..e71e98d 100644 --- a/docs/_static/analogvnn-logo-square-black.svg +++ b/docs/_static/analogvnn-logo-square-black.svg @@ -1,35 +1,39 @@ - - + + + + + + + + + + - \ No newline at end of file diff --git a/docs/_static/analogvnn-logo-square-white.svg b/docs/_static/analogvnn-logo-square-white.svg index 994efac..6621c76 100644 --- a/docs/_static/analogvnn-logo-square-white.svg +++ b/docs/_static/analogvnn-logo-square-white.svg @@ -1,35 +1,39 @@ - - + + + + + + + + + + - \ No newline at end of file diff --git a/docs/_static/analogvnn-logo-wide-black.svg b/docs/_static/analogvnn-logo-wide-black.svg index 8accb63..28187ea 100644 --- a/docs/_static/analogvnn-logo-wide-black.svg +++ b/docs/_static/analogvnn-logo-wide-black.svg @@ -1,48 +1,61 @@ - - + + + + + + + + + + + + + + + + + + + + + + + - \ No newline at end of file diff --git a/docs/_static/analogvnn-logo-wide-white.svg b/docs/_static/analogvnn-logo-wide-white.svg index 7c265ce..6f63f77 100644 --- a/docs/_static/analogvnn-logo-wide-white.svg +++ b/docs/_static/analogvnn-logo-wide-white.svg @@ -1,48 +1,61 @@ - - + + + + + + + + + + + + + + + + + + + + + + + - \ No newline at end of file diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 70fc2c5..75425b7 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,3 +1,3 @@ -.sidebar-brand-text{ +.sidebar-brand-text { display: none; } \ No newline at end of file diff --git a/docs/extra_classes.md b/docs/extra_classes.md index 4b89921..66300fa 100644 --- a/docs/extra_classes.md +++ b/docs/extra_classes.md @@ -93,6 +93,7 @@ p & : \ p \gt x ``` where: + - p, q ∈ ℜ (p ≤ q, Default value for photonics p = −1 and q = 1) ## Noise diff --git a/docs/inner_workings.md b/docs/inner_workings.md index ca2e1c9..0656556 100644 --- a/docs/inner_workings.md +++ b/docs/inner_workings.md @@ -15,13 +15,14 @@ the parameter of layer of `Parameter` class to `PseudoParameters`. layer to get parameterized data PyTorch's ParameterizedParameters vs AnalogVNN's PseudoParameters: + - Similarity (Forward or Parameterizing the data): > Data → ParameterizingModel → Parameterized Data - Difference (Backward or Gradient Calculations): - - ParameterizedParameters - > Parameterized Data → ParameterizingModel → Data - - PseudoParameters - > Parameterized Data → Data + - ParameterizedParameters + > Parameterized Data → ParameterizingModel → Data + - PseudoParameters + > Parameterized Data → Data So, by using `PseudoParameters` class the gradients of the parameter are calculated in such a way that the ParameterizingModel was never present. diff --git a/docs/install.md b/docs/install.md index 90d4f59..3d8e707 100644 --- a/docs/install.md +++ b/docs/install.md @@ -34,23 +34,23 @@ OR Install the required dependencies: - PyTorch - - Manual installation required: [https://pytorch.org/](https://pytorch.org/) + - Manual installation required: [https://pytorch.org/](https://pytorch.org/) - dataclasses - scipy - numpy - networkx - (optional) tensorboard - - For using tensorboard to visualize the network, with class - {py:class}`analogvnn.utils.TensorboardModelLog.TensorboardModelLog` + - For using tensorboard to visualize the network, with class + {py:class}`analogvnn.utils.TensorboardModelLog.TensorboardModelLog` - (optional) torchinfo - - For adding summary to tensorboard by using - {py:func}`analogvnn.utils.TensorboardModelLog.TensorboardModelLog.add_summary` + - For adding summary to tensorboard by using + {py:func}`analogvnn.utils.TensorboardModelLog.TensorboardModelLog.add_summary` - (optional) graphviz - - For saving and rendering forward and backward graphs using - {py:func}`analogvnn.graph.AcyclicDirectedGraph.AcyclicDirectedGraph.render` + - For saving and rendering forward and backward graphs using + {py:func}`analogvnn.graph.AcyclicDirectedGraph.AcyclicDirectedGraph.render` - (optional) python-graphviz - - For saving and rendering forward and backward graphs using - {py:func}`analogvnn.graph.AcyclicDirectedGraph.AcyclicDirectedGraph.render` + - For saving and rendering forward and backward graphs using + {py:func}`analogvnn.graph.AcyclicDirectedGraph.AcyclicDirectedGraph.render`

diff --git a/requirements-test.txt b/requirements-test.txt index bba026e..f67ad55 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -6,3 +6,6 @@ flake8-bugbear flake8-comprehensions flake8-executable flake8-coding +flake8-return +flake8-noreturn +flake8-deprecated diff --git a/sample_code.py b/sample_code.py index 050959e..20a106d 100644 --- a/sample_code.py +++ b/sample_code.py @@ -16,6 +16,7 @@ def load_vision_dataset(dataset, path, batch_size, is_cuda=False, grayscale=True): """ + Loads a vision dataset with optional grayscale conversion and CUDA support. Args: @@ -28,6 +29,7 @@ def load_vision_dataset(dataset, path, batch_size, is_cuda=False, grayscale=True Returns: A tuple containing the train and test data loaders, the input shape, and a tuple of class labels. """ + dataset_kwargs = { 'batch_size': batch_size, 'shuffle': True @@ -69,6 +71,7 @@ def cross_entropy_accuracy(output, target) -> float: Returns: float: accuracy from 0 to 1 """ + _, preds = torch.max(output.data, 1) correct = (preds == target).sum().item() return correct / len(output) @@ -89,6 +92,7 @@ def __init__(self, activation_class, norm_class, precision_class, precision, noi (i.e., the probability that the digital values transmitted and detected are different after passing through the analog channel). """ + super(LinearModel, self).__init__() self.activation_class = activation_class @@ -112,6 +116,7 @@ def add_layer(self, layer): Args: layer (BaseLayer): digital layer module """ + self.all_layers.append(self.norm_class()) self.all_layers.append(self.precision_class(precision=self.precision)) self.all_layers.append(self.noise_class(leakage=self.leakage, precision=self.precision)) @@ -137,6 +142,7 @@ def __init__(self, norm_class, precision_class, precision, noise_class, leakage) (i.e., the probability that the digital values transmitted and detected are different after passing through the analog channel). """ + super(WeightModel, self).__init__() self.all_layers = [] @@ -150,6 +156,7 @@ def __init__(self, norm_class, precision_class, precision, noise_class, leakage) def run_linear3_model(): """The main function to train photonics image classifier with 3 linear/dense nn for MNIST dataset.""" + is_cpu_cuda.use_cuda_if_available() torch.backends.cudnn.benchmark = True torch.manual_seed(0) diff --git a/sample_code_with_logs.py b/sample_code_with_logs.py index 23ffdce..c0f271e 100644 --- a/sample_code_with_logs.py +++ b/sample_code_with_logs.py @@ -32,6 +32,7 @@ def load_vision_dataset(dataset, path, batch_size, is_cuda=False, grayscale=True Returns: A tuple containing the train and test data loaders, the input shape, and a tuple of class labels. """ + dataset_kwargs = { 'batch_size': batch_size, 'shuffle': True @@ -73,6 +74,7 @@ def cross_entropy_accuracy(output, target) -> float: Returns: float: accuracy from 0 to 1 """ + _, preds = torch.max(output.data, 1) correct = (preds == target).sum().item() return correct / len(output) @@ -93,6 +95,7 @@ def __init__(self, activation_class, norm_class, precision_class, precision, noi (i.e., the probability that the digital values transmitted and detected are different after passing through the analog channel). """ + super(LinearModel, self).__init__() self.activation_class = activation_class @@ -116,6 +119,7 @@ def add_layer(self, layer): Args: layer (BaseLayer): digital layer module """ + self.all_layers.append(self.norm_class()) self.all_layers.append(self.precision_class(precision=self.precision)) self.all_layers.append(self.noise_class(leakage=self.leakage, precision=self.precision)) @@ -141,6 +145,7 @@ def __init__(self, norm_class, precision_class, precision, noise_class, leakage) (i.e., the probability that the digital values transmitted and detected are different after passing through the analog channel). """ + super(WeightModel, self).__init__() self.all_layers = [] @@ -154,6 +159,7 @@ def __init__(self, norm_class, precision_class, precision, noise_class, leakage) def run_linear3_model(): """The main function to train photonics image classifier with 3 linear/dense nn for MNIST dataset.""" + is_cpu_cuda.use_cuda_if_available() torch.backends.cudnn.benchmark = True torch.manual_seed(0) diff --git a/unit_tests/test_model_graphs.py b/unit_tests/test_model_graphs.py index 1d0d115..f695523 100644 --- a/unit_tests/test_model_graphs.py +++ b/unit_tests/test_model_graphs.py @@ -12,15 +12,19 @@ l1 = nn.Linear(1, 1, bias=False) l1.weight.data = torch.ones_like(l1.weight.data) * 2 + def l2(*x): return torch.add(*x), torch.sub(*x) + def l3(x, y): return {"a": torch.sub(x, y), "b": torch.add(x, y)} + def l4(x, y, z, a, b): return ((x + y) + (a + b)) + z + def l5(x): return {"c": x * 0.5} diff --git a/unit_tests/test_pseudo_parameter.py b/unit_tests/test_pseudo_parameter.py index 6291243..726d5ef 100644 --- a/unit_tests/test_pseudo_parameter.py +++ b/unit_tests/test_pseudo_parameter.py @@ -43,13 +43,16 @@ def __init__(self): def forward(self, x): return x + (torch.ones_like(x) * self.weight) + class Symmetric(BackwardIdentity, Model): def forward(self, x): return torch.rand((1, x.size()[0])) @ x @ torch.rand((x.size()[1], 1)) + def pstr(s): return str(s).replace(" ", "").replace("\n", "") + model = Layer() parametrization = Symmetric() # parametrization.eval() From 1d96912da629b3677e8a13d1f99929863dc7c1b4 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Fri, 13 Jan 2023 23:23:23 -0500 Subject: [PATCH 13/16] updated: requirements Signed-off-by: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> --- requirements-all.txt | 4 ++++ requirements-dev.txt => requirements/requirements-dev.txt | 0 requirements-docs.txt => requirements/requirements-docs.txt | 0 requirements-test.txt => requirements/requirements-test.txt | 0 4 files changed, 4 insertions(+) create mode 100644 requirements-all.txt rename requirements-dev.txt => requirements/requirements-dev.txt (100%) rename requirements-docs.txt => requirements/requirements-docs.txt (100%) rename requirements-test.txt => requirements/requirements-test.txt (100%) diff --git a/requirements-all.txt b/requirements-all.txt new file mode 100644 index 0000000..617af85 --- /dev/null +++ b/requirements-all.txt @@ -0,0 +1,4 @@ +-r requirements.txt +-r requirements/requirements-docs.txt +-r requirements/requirements-test.txt +-r requirements/requirements-dev.txt diff --git a/requirements-dev.txt b/requirements/requirements-dev.txt similarity index 100% rename from requirements-dev.txt rename to requirements/requirements-dev.txt diff --git a/requirements-docs.txt b/requirements/requirements-docs.txt similarity index 100% rename from requirements-docs.txt rename to requirements/requirements-docs.txt diff --git a/requirements-test.txt b/requirements/requirements-test.txt similarity index 100% rename from requirements-test.txt rename to requirements/requirements-test.txt From a67c8967c473b29f1d91afdfe8d58dcc3cc9691b Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sat, 14 Jan 2023 00:11:58 -0500 Subject: [PATCH 14/16] correction: registered_modules to registered_children Signed-off-by: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> --- analogvnn/nn/module/FullSequential.py | 2 +- analogvnn/nn/module/Layer.py | 34 +++---- analogvnn/nn/module/Model.py | 9 +- analogvnn/nn/module/Sequential.py | 131 ++++---------------------- sample_code.py | 9 +- sample_code_with_logs.py | 12 ++- 6 files changed, 46 insertions(+), 151 deletions(-) diff --git a/analogvnn/nn/module/FullSequential.py b/analogvnn/nn/module/FullSequential.py index 0c7e3a0..9411383 100644 --- a/analogvnn/nn/module/FullSequential.py +++ b/analogvnn/nn/module/FullSequential.py @@ -22,6 +22,6 @@ def compile(self, device: Optional[torch.device] = None, layer_data: bool = True Returns: FullSequential: self """ - arr = [self.graphs.INPUT, *list(self._runtime_module_list.values()), self.graphs.OUTPUT] + arr = [self.graphs.INPUT, *list(self.registered_children()), self.graphs.OUTPUT] self.graphs.backward_graph.add_connection(*reversed(arr)) return super().compile(device, layer_data) diff --git a/analogvnn/nn/module/Layer.py b/analogvnn/nn/module/Layer.py index 846144f..cae872b 100644 --- a/analogvnn/nn/module/Layer.py +++ b/analogvnn/nn/module/Layer.py @@ -191,23 +191,19 @@ def set_backward_function(self, backward_class: Union[Callable, BackwardModule, return self - def named_registered_modules( + def named_registered_children( self, - memo: Optional[Set[nn.Module]] = None, - prefix: str = '', - remove_duplicate: bool = True + memo: Optional[Set[nn.Module]] = None ) -> Iterator[Tuple[str, nn.Module]]: - """Returns an iterator over all registered modules in the network, yielding - both the name of the module and the module itself. + r"""Returns an iterator over immediate registered children modules, yielding both + the name of the module. Args: memo: a memo to store the set of modules already added to the result - prefix: a prefix that will be added to the name of the module - remove_duplicate: whether to remove the duplicated module instances in the result - or not + Yields: - (str, Module): Tuple of name and module + (str, Module): Tuple containing a name and child module Note: Duplicate modules are returned only once. In the following @@ -216,21 +212,17 @@ def named_registered_modules( if memo is None: memo = set() - if self.backward_function != self: - memo.add(self.backward_function) + memo.add(self.backward_function) + memo.add(self) - for name, module in super(Layer, self).named_modules( - memo=memo, - prefix=prefix, - remove_duplicate=remove_duplicate - ): - if module is self: + for name, module in self.named_children(): + if module in memo: continue yield name, module - def registered_modules(self) -> Iterator[nn.Module]: - """Returns an iterator over registered modules under self. + def registered_children(self) -> Iterator[nn.Module]: + r"""Returns an iterator over immediate registered children modules. Yields: nn.Module: a module in the network @@ -239,7 +231,7 @@ def registered_modules(self) -> Iterator[nn.Module]: Duplicate modules are returned only once. In the following example, ``l`` will be returned only once. """ - for _, module in self.named_registered_modules(): + for _, module in self.named_registered_children(): yield module def _forward_wrapper(self, function: Callable) -> Callable: diff --git a/analogvnn/nn/module/Model.py b/analogvnn/nn/module/Model.py index ca5e1a2..4ed6f4f 100644 --- a/analogvnn/nn/module/Model.py +++ b/analogvnn/nn/module/Model.py @@ -112,20 +112,15 @@ def use_autograd_graph(self, use_autograd_graph: bool): """ self.graphs.use_autograd_graph = use_autograd_graph - def named_registered_modules( + def named_registered_children( self, memo: Optional[Set[nn.Module]] = None, - prefix: str = '', - remove_duplicate: bool = True ) -> Iterator[Tuple[str, nn.Module]]: """Returns an iterator over registered modules under self, yielding both the name of the module and the module itself. Args: memo: a memo to store the set of modules already added to the result - prefix: a prefix that will be added to the name of the module - remove_duplicate: whether to remove the duplicated module instances in the result - or not Yields: (str, nn.Module): Tuple of name and module @@ -140,7 +135,7 @@ def named_registered_modules( memo.add(self.optimizer) memo.add(self.loss_function) memo.add(self.accuracy_function) - return super(Model, self).named_registered_modules(memo=memo, prefix=prefix, remove_duplicate=remove_duplicate) + return super(Model, self).named_registered_children(memo=memo) def compile(self, device: Optional[torch.device] = None, layer_data: bool = True): """Compile the model. diff --git a/analogvnn/nn/module/Sequential.py b/analogvnn/nn/module/Sequential.py index 1ec0e67..e8bc45b 100644 --- a/analogvnn/nn/module/Sequential.py +++ b/analogvnn/nn/module/Sequential.py @@ -1,13 +1,9 @@ from __future__ import annotations -import operator -from collections import OrderedDict -from itertools import islice -from typing import Iterator, TypeVar, Union, Dict, Optional +from typing import TypeVar, Optional import torch from torch import nn -from torch.nn import Module from analogvnn.nn.module.Model import Model @@ -16,22 +12,23 @@ __all__ = ['Sequential'] -class Sequential(Model): - """A sequential model. +class Sequential(Model, nn.Sequential): + """Base class for all sequential models.""" - Attributes: - _runtime_module_list (OrderedDict[str, nn.Module]): The ordered dictionary of the modules. - """ - - def __init__(self, *args): - """Initialize the model. + def __call__(self, *args, **kwargs): + """Call the model. Args: - *args: The modules to add. + *args: The input. + **kwargs: The input. + + Returns: + torch.Tensor: The output of the model. """ - super(Sequential, self).__init__() - self._runtime_module_list: Dict[str, Optional[Module]] = OrderedDict() - self.add_sequence(*args) + if not self._compiled: + self.compile() + + return super().__call__(*args, **kwargs) def compile(self, device: Optional[torch.device] = None, layer_data: bool = True): """Compile the model and add forward graph. @@ -43,7 +40,7 @@ def compile(self, device: Optional[torch.device] = None, layer_data: bool = True Returns: Sequential: self """ - arr = [self.graphs.INPUT, *list(self._runtime_module_list.values()), self.graphs.OUTPUT] + arr = [self.graphs.INPUT, *list(self.registered_children()), self.graphs.OUTPUT] self.graphs.forward_graph.add_connection(*arr) return super().compile(device, layer_data) @@ -53,100 +50,4 @@ def add_sequence(self, *args): Args: *args (nn.Module): The modules to add. """ - if len(args) == 1 and isinstance(args[0], OrderedDict): - for key, module in args[0].items(): - self._add_run_module(key, module) - else: - for idx, module in enumerate(args): - self._add_run_module(str(idx), module) - - def _add_run_module(self, name: str, module: Optional[Module]): - """Add a module to the forward graph of model. - - Args: - name (str): The name of the module. - module (nn.Module): The module to add. - """ - self.add_module(name, module) - self._runtime_module_list[name] = module - return self - - def _get_item_by_idx(self, iterator, idx) -> T: - """Get the idx-th item of the iterator. - - Args: - iterator (Iterator): The iterator. - idx (int): The index of the item. - - Returns: - T: The idx-th item of the iterator. - """ - size = len(self) - idx = operator.index(idx) - if not -size <= idx < size: - raise IndexError('index {} is out of range'.format(idx)) - idx %= size - return next(islice(iterator, idx, None)) - - def __getitem__(self, idx) -> Union[Sequential, T]: - """Get the idx-th module of the model. - - Args: - idx (int): The index of the module. - - Returns: - Union[Sequential, T]: The idx-th module of the model. - """ - if isinstance(idx, slice): - return self.__class__(OrderedDict(list(self._runtime_module_list.items())[idx])) - else: - return self._get_item_by_idx(self._runtime_module_list.values(), idx) - - def __setitem__(self, idx: int, module: nn.Module) -> None: - """Set the idx-th module of the model. - - Args: - idx (int): The index of the module. - module (nn.Module): The module to set. - """ - key: str = self._get_item_by_idx(self._runtime_module_list.keys(), idx) - return setattr(self, key, module) - - def __delitem__(self, idx: Union[slice, int]) -> None: - """Remove the idx-th module from the model. - - Args: - idx (Union[slice, int]): The index of the module. - """ - if isinstance(idx, slice): - for key in list(self._runtime_module_list.keys())[idx]: - delattr(self, key) - else: - key = self._get_item_by_idx(self._runtime_module_list.keys(), idx) - delattr(self, key) - - def __len__(self) -> int: - """Return the number of modules in the model. - - Returns: - int: The number of modules in the model. - """ - return len(self._runtime_module_list) - - def __dir__(self): - """Return the list of attributes of the module. - - Returns: - list: The list of attributes of the module. - """ - keys = super(Sequential, self).__dir__() - keys = [key for key in keys if not key.isdigit()] - return keys - - def __iter__(self) -> Iterator[nn.Module]: - """Return an iterator over the modules of the model. - - Returns: - Iterator[nn.Module]: An iterator over the modules of the model. - """ - return iter(self._runtime_module_list.values()) + return self.extend(args) diff --git a/sample_code.py b/sample_code.py index 050959e..f0e3ea4 100644 --- a/sample_code.py +++ b/sample_code.py @@ -184,15 +184,18 @@ def run_linear3_model(): leakage=0.5 ) + # Parametrizing Parameters of the Models + PseudoParameter.parametrize_module(nn_model, transformation=weight_model) + # Setting Model Parameters nn_model.loss_function = nn.CrossEntropyLoss() nn_model.accuracy_function = cross_entropy_accuracy + nn_model.optimizer = optim.Adam(params=nn_model.parameters()) + + # Compile Model nn_model.compile(device=device) weight_model.compile(device=device) - PseudoParameter.parametrize_module(nn_model, transformation=weight_model) - nn_model.optimizer = optim.Adam(params=nn_model.parameters()) - # Training print('Starting Training...') for epoch in range(10): diff --git a/sample_code_with_logs.py b/sample_code_with_logs.py index 23ffdce..60ea5c2 100644 --- a/sample_code_with_logs.py +++ b/sample_code_with_logs.py @@ -192,15 +192,19 @@ def run_linear3_model(): leakage=0.5 ) + # Parametrizing Parameters of the Models + PseudoParameter.parametrize_module(nn_model, transformation=weight_model) + # Setting Model Parameters nn_model.loss_function = nn.CrossEntropyLoss() nn_model.accuracy_function = cross_entropy_accuracy + nn_model.optimizer = optim.Adam(params=nn_model.parameters()) + + # Compile Model nn_model.compile(device=device) weight_model.compile(device=device) - PseudoParameter.parametrize_module(nn_model, transformation=weight_model) - nn_model.optimizer = optim.Adam(params=nn_model.parameters()) - + # Creating Tensorboard for Visualization and Saving the Model nn_model.create_tensorboard(str(data_path.joinpath('tensorboard'))) print('Saving Summary and Graphs...') @@ -240,7 +244,7 @@ def run_linear3_model(): # Training print('Starting Training...') - for epoch in range(10): + for epoch in range(1): train_loss, train_accuracy = nn_model.train_on(train_loader, epoch=epoch) test_loss, test_accuracy = nn_model.test_on(test_loader, epoch=epoch) From 6faa8c1ee2a10c5f5c4db0479ff9dfc44cf44ff3 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sat, 14 Jan 2023 00:17:35 -0500 Subject: [PATCH 15/16] order change Signed-off-by: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> --- analogvnn/nn/module/Layer.py | 2 +- sample_code_with_logs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/analogvnn/nn/module/Layer.py b/analogvnn/nn/module/Layer.py index 311711d..6a3ca65 100644 --- a/analogvnn/nn/module/Layer.py +++ b/analogvnn/nn/module/Layer.py @@ -222,8 +222,8 @@ def named_registered_children( if memo is None: memo = set() - memo.add(self.backward_function) memo.add(self) + memo.add(self.backward_function) for name, module in self.named_children(): if module in memo: diff --git a/sample_code_with_logs.py b/sample_code_with_logs.py index 9bcb5c4..acafa9c 100644 --- a/sample_code_with_logs.py +++ b/sample_code_with_logs.py @@ -250,7 +250,7 @@ def run_linear3_model(): # Training print('Starting Training...') - for epoch in range(1): + for epoch in range(10): train_loss, train_accuracy = nn_model.train_on(train_loader, epoch=epoch) test_loss, test_accuracy = nn_model.test_on(test_loader, epoch=epoch) From 1cd2bdeca66dc681052687d34fb0970f13742b74 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sat, 14 Jan 2023 01:41:26 -0500 Subject: [PATCH 16/16] flake8 and minor corrections Signed-off-by: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> --- analogvnn/graph/AcyclicDirectedGraph.py | 4 ++-- analogvnn/nn/module/Layer.py | 3 +-- analogvnn/nn/module/Sequential.py | 2 +- pyproject.toml | 2 +- requirements.txt | 8 ++++---- requirements/requirements-test.txt | 2 +- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/analogvnn/graph/AcyclicDirectedGraph.py b/analogvnn/graph/AcyclicDirectedGraph.py index 7f14b19..f333918 100644 --- a/analogvnn/graph/AcyclicDirectedGraph.py +++ b/analogvnn/graph/AcyclicDirectedGraph.py @@ -131,8 +131,8 @@ def add_edge( self.graph.nodes[v_of_edge]['fillcolor'] = 'lightblue' return self - @staticmethod - def check_edge_parameters( # noqa: C901 + @staticmethod # noqa: C901 + def check_edge_parameters( in_arg: Union[None, int, bool], in_kwarg: Union[None, str, bool], out_arg: Union[None, int, bool], diff --git a/analogvnn/nn/module/Layer.py b/analogvnn/nn/module/Layer.py index 6a3ca65..f73630e 100644 --- a/analogvnn/nn/module/Layer.py +++ b/analogvnn/nn/module/Layer.py @@ -204,8 +204,7 @@ def named_registered_children( self, memo: Optional[Set[nn.Module]] = None ) -> Iterator[Tuple[str, nn.Module]]: - """Returns an iterator over immediate registered children modules, yielding both - the name of the module. + """Returns an iterator over immediate registered children modules. Args: memo: a memo to store the set of modules already added to the result diff --git a/analogvnn/nn/module/Sequential.py b/analogvnn/nn/module/Sequential.py index c16d49b..e06afb5 100644 --- a/analogvnn/nn/module/Sequential.py +++ b/analogvnn/nn/module/Sequential.py @@ -51,5 +51,5 @@ def add_sequence(self, *args): Args: *args (nn.Module): The modules to add. """ - + return self.extend(args) diff --git a/pyproject.toml b/pyproject.toml index 0f24fc3..51a3cb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,7 +102,7 @@ flake8 = [ "flake8-executable", "flake8-coding", "flake8-return", - "flake8-noreturn", +# "flake8-noreturn; python_version >= '3.8'", "flake8-deprecated", ] dev = [ diff --git a/requirements.txt b/requirements.txt index d5668b2..464c65c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,12 +5,12 @@ torchaudio numpy scipy networkx -importlib-metadata<5.0.0,>=2.0.0; python_version < "3.8" +importlib-metadata<5.0.0,>=2.0.0; python_version < '3.8' # Full -tensorflow -tensorboard +tensorflow>=2.0.0 +tensorboard>=2.0.0 torchinfo pillow -# conda install python-graphviz graphviz +# conda install graphviz graphviz diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt index f67ad55..54c409a 100644 --- a/requirements/requirements-test.txt +++ b/requirements/requirements-test.txt @@ -7,5 +7,5 @@ flake8-comprehensions flake8-executable flake8-coding flake8-return -flake8-noreturn +# flake8-noreturn>=1.0.1; python_version >= '3.8' flake8-deprecated