Update the Neptune logger (#2881)

* added output handlers, model checkpoint handler, added mnist example, added initial tests file

* added exp link to examples, added tests

* added neptune do docs

* fixed test

* fixed imports

* added neptune-client to test dependencies

* fixed missing package message

* dropped model checkpoing handler

* updated experiment link

* dropped __futures__ print_function

* added NeptuneSaver and tests

* autopep8 fix

* updated token to anonymous user neptuner

* updated experiment link

* updated token to 'ANONYMOUS'

* updated examples, fixed tests

* autopep8 fix

* fixed serializable test

* fixed serializable model test

* local

* autopep8 fix

* added self.experiment back

* Fix after merge

* Update Neptune integration for compatibility with recent changes in Neptune client

* Track the version of Ignite in Neptune

* Ignore the future deprecation warnings when importing neptune.new

* autopep8 fix

* Make INTEGRATION_VERSION_KEY private

* Add type annotations

* Add type annotation to kwargs

* Use >= in requirements

* Fix failing test

---------

Co-authored-by: jakubczakon <czakon.jakub@gmail.com>
Co-authored-by: AutoPEP8 <>

Author: twolodzko

ignite/contrib/handlers/neptune_logger.py

@@ -6,6 +6,7 @@
 from torch.optim import Optimizer
 
 import ignite.distributed as idist
+from ignite import __version__
 from ignite.contrib.handlers.base_logger import (
     BaseLogger,
     BaseOptimizerParamsHandler,
@@ -26,6 +27,8 @@
     "global_step_from_engine",
 ]
 
+_INTEGRATION_VERSION_KEY = "source_code/integrations/neptune-pytorch-ignite"
+
 
 class NeptuneLogger(BaseLogger):
     """
@@ -42,24 +45,7 @@ class NeptuneLogger(BaseLogger):
            "namespace/project_name" for example "tom/minst-classification".
            If None, the value of NEPTUNE_PROJECT environment variable will be taken.
            You need to create the project in https://neptune.ai first.
-        offline_mode: Optional default False. If offline_mode=True no logs will be send to neptune.
-           Usually used for debug purposes.
-        experiment_name: Optional. Editable name of the experiment.
-           Name is displayed in the experiment’s Details (Metadata section) and in experiments view as a column.
-        upload_source_files: Optional. List of source files to be uploaded.
-           Must be list of str or single str. Uploaded sources are displayed in the experiment’s Source code tab.
-           If None is passed, Python file from which experiment was created will be uploaded.
-           Pass empty list (`[]`) to upload no files. Unix style pathname pattern expansion is supported.
-           For example, you can pass `*.py` to upload all python source files from the current directory.
-           For recursion lookup use `**/*.py` (for Python 3.5 and later). For more information see glob library.
-        params: Optional. Parameters of the experiment. After experiment creation params are read-only.
-           Parameters are displayed in the experiment’s Parameters section and each key-value pair can be
-           viewed in experiments view as a column.
-        properties: Optional default is `{}`. Properties of the experiment.
-           They are editable after experiment is created. Properties are displayed in the experiment’s Details and
-           each key-value pair can be viewed in experiments view as a column.
-        tags: Optional default `[]`. Must be list of str. Tags of the experiment.
-           Tags are displayed in the experiment’s Details and can be viewed in experiments view as a column.
+        **kwargs: Other arguments to be passed to Neptune's `init_run`.
 
     Examples:
         .. code-block:: python
@@ -71,8 +57,8 @@ class NeptuneLogger(BaseLogger):
 
             npt_logger = NeptuneLogger(
                 api_token="ANONYMOUS",
-                project_name="shared/pytorch-ignite-integration",
-                experiment_name="cnn-mnist", # Optional,
+                project="shared/pytorch-ignite-integration",
+                name="cnn-mnist", # Optional,
                 params={"max_epochs": 10}, # Optional,
                 tags=["pytorch-ignite","minst"] # Optional
             )
@@ -153,8 +139,8 @@ def score_function(engine):
             # We are using the api_token for the anonymous user neptuner but you can use your own.
 
             with NeptuneLogger(api_token="ANONYMOUS",
-                               project_name="shared/pytorch-ignite-integration",
-                               experiment_name="cnn-mnist", # Optional,
+                               project="shared/pytorch-ignite-integration",
+                               name="cnn-mnist", # Optional,
                                params={"max_epochs": 10}, # Optional,
                                tags=["pytorch-ignite","mnist"] # Optional
                                ) as npt_logger:
@@ -171,39 +157,44 @@ def score_function(engine):
     """
 
     def __getattr__(self, attr: Any) -> Any:
+        return getattr(self.experiment, attr)
 
-        import neptune
+    def __getitem__(self, key: str) -> Any:
+        return self.experiment[key]
 
-        return getattr(neptune, attr)
+    def __setitem__(self, key: str, val: Any) -> Any:
+        self.experiment[key] = val
+
+    def __init__(self, api_token: Optional[str] = None, project: Optional[str] = None, **kwargs: Any) -> None:
+        import warnings
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
         try:
-            import neptune
+            try:
+                # neptune-client<1.0.0 package structure
+                with warnings.catch_warnings():
+                    # ignore the deprecation warnings
+                    warnings.simplefilter("ignore")
+                    import neptune.new as neptune
+            except ImportError:
+                # neptune>=1.0.0 package structure
+                import neptune
         except ImportError:
             raise ModuleNotFoundError(
-                "This contrib module requires neptune-client to be installed. "
-                "You may install neptune with command: \n pip install neptune-client \n"
-            )
-
-        if kwargs.get("offline_mode", False):
-            self.mode = "offline"
-            neptune.init(
-                project_qualified_name="dry-run/project",
-                backend=neptune.OfflineBackend(),
+                "This contrib module requires neptune client to be installed. "
+                "You may install neptune with command: \n pip install neptune \n"
             )
-        else:
-            self.mode = "online"
-            neptune.init(api_token=kwargs.get("api_token"), project_qualified_name=kwargs.get("project_name"))
 
-        kwargs["name"] = kwargs.pop("experiment_name", None)
-        self._experiment_kwargs = {
-            k: v for k, v in kwargs.items() if k not in ["api_token", "project_name", "offline_mode"]
-        }
+        run = neptune.init_run(
+            api_token=api_token,
+            project=project,
+            **kwargs,
+        )
+        run[_INTEGRATION_VERSION_KEY] = __version__
 
-        self.experiment = neptune.create_experiment(**self._experiment_kwargs)
+        self.experiment = run
 
     def close(self) -> None:
-        self.stop()
+        self.experiment.stop()
 
     def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler":
         return OutputHandler(*args, **kwargs)
@@ -355,7 +346,7 @@ def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str,
             )
 
         for key, value in metrics.items():
-            logger.log_metric(key, x=global_step, y=value)
+            logger[key].append(value, step=global_step)
 
 
 class OptimizerParamsHandler(BaseOptimizerParamsHandler):
@@ -412,7 +403,7 @@ def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str,
         }
 
         for k, v in params.items():
-            logger.log_metric(k, x=global_step, y=v)
+            logger[k].append(v, step=global_step)
 
 
 class WeightsScalarHandler(BaseWeightsScalarHandler):
@@ -515,11 +506,8 @@ def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str,
                 continue
 
             name = name.replace(".", "/")
-            logger.log_metric(
-                f"{tag_prefix}weights_{self.reduction.__name__}/{name}",
-                x=global_step,
-                y=self.reduction(p.data),
-            )
+            key = f"{tag_prefix}weights_{self.reduction.__name__}/{name}"
+            logger[key].append(self.reduction(p.data), step=global_step)
 
 
 class GradsScalarHandler(BaseWeightsScalarHandler):
@@ -622,9 +610,8 @@ def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str,
                 continue
 
             name = name.replace(".", "/")
-            logger.log_metric(
-                f"{tag_prefix}grads_{self.reduction.__name__}/{name}", x=global_step, y=self.reduction(p.grad)
-            )
+            key = f"{tag_prefix}grads_{self.reduction.__name__}/{name}"
+            logger[key].append(self.reduction(p.grad), step=global_step)
 
 
 class NeptuneSaver(BaseSaveHandler):
@@ -693,8 +680,8 @@ def __call__(self, checkpoint: Mapping, filename: str, metadata: Optional[Mappin
             # we can not use tmp.name to open tmp.file twice on Win32
             # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile
             torch.save(checkpoint, tmp.file)
-            self._logger.log_artifact(tmp.name, filename)
+            self._logger[filename].upload(tmp.name)
 
     @idist.one_rank_only(with_barrier=True)
     def remove(self, filename: str) -> None:
-        self._logger.delete_artifacts(filename)
+        self._logger.delete_files(filename)

requirements-dev.txt

@@ -21,7 +21,7 @@ polyaxon
 polyaxon-client
 wandb
 mlflow
-neptune-client==0.16.9
+neptune-client>=0.16.17
 tensorboard
 torchvision
 pynvml

tests/ignite/contrib/engines/test_common.py

@@ -561,7 +561,7 @@ def test_setup_clearml_logging():
 def test_setup_neptune_logging(dirname):
     npt_logger = _test_setup_logging(
         setup_logging_fn=setup_neptune_logging,
-        kwargs_dict={"offline_mode": True},
+        kwargs_dict={"mode": "offline"},
         output_handler_cls=handlers.neptune_logger.OutputHandler,
         opt_params_handler_cls=handlers.neptune_logger.OptimizerParamsHandler,
         with_eval=False,
@@ -570,7 +570,7 @@ def test_setup_neptune_logging(dirname):
     npt_logger.close()
     npt_logger = _test_setup_logging(
         setup_logging_fn=setup_neptune_logging,
-        kwargs_dict={"offline_mode": True},
+        kwargs_dict={"mode": "offline"},
         output_handler_cls=handlers.neptune_logger.OutputHandler,
         opt_params_handler_cls=handlers.neptune_logger.OptimizerParamsHandler,
         with_eval=True,