Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] Refactor FedRunner, optimize trainer module and optimize CI #415

Merged
merged 22 commits into from
Nov 16, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix docstring in app
  • Loading branch information
rayrayraykk committed Nov 2, 2022
commit ab345a56238db4c401398635fc1ce07b34d9e637
6 changes: 5 additions & 1 deletion doc/source/cv.rst
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ Federated Computer Vision Module References
federatedscope.cv.dataset
-----------------------

.. automodule:: federatedscope.cv.dataset
.. automodule:: federatedscope.cv.dataset.leaf
:members:
:private-members:

.. automodule:: federatedscope.cv.dataset.leaf_cv
:members:
:private-members:

Expand Down
22 changes: 14 additions & 8 deletions federatedscope/cv/dataloader/dataloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,20 @@


def load_cv_dataset(config=None):
r"""
return {
'client_id': {
'train': DataLoader(),
'test': DataLoader(),
'val': DataLoader()
}
}
"""
Return the dataset of ``femnist`` or ``celeba``.

Args:
config: configurations for FL, see ``federatedscope.core.configs``

Returns:
FL dataset dict, with ``client_id`` as key.

Note:
``load_cv_dataset()`` will return a dict as shown below:
```
{'client_id': {'train': dataset, 'test': dataset, 'val': dataset}}
```
"""
splits = config.data.splits

Expand Down
5 changes: 3 additions & 2 deletions federatedscope/cv/dataset/leaf.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ def is_exists(path, names):


class LEAF(Dataset):
"""Base class for LEAF dataset from "LEAF: A Benchmark for Federated Settings"
"""
Base class for LEAF dataset from "LEAF: A Benchmark for Federated Settings"

Arguments:
root (str): root path.
Expand Down Expand Up @@ -91,7 +92,7 @@ def process(self):

class LocalDataset(Dataset):
"""
Convert data list to torch Dataset to save memory usage.
Convert data list to torch Dataset to save memory usage.
"""
def __init__(self,
Xs,
Expand Down
3 changes: 3 additions & 0 deletions federatedscope/cv/trainer/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@


class CVTrainer(GeneralTorchTrainer):
"""
``CVTrainer`` is the same as ``core.trainers.GeneralTorchTrainer``.
"""
pass


Expand Down
23 changes: 15 additions & 8 deletions federatedscope/nlp/dataloader/dataloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,21 @@


def load_nlp_dataset(config=None):
r"""
return {
'client_id': {
'train': DataLoader(),
'test': DataLoader(),
'val': DataLoader()
}
}
"""
Return the dataset of ``shakespeare``, ``subreddit``, ``twitter``, \
or ``synthetic``.

Args:
config: configurations for FL, see ``federatedscope.core.configs``

Returns:
FL dataset dict, with ``client_id`` as key.

Note:
``load_nlp_dataset()`` will return a dict as shown below:
```
{'client_id': {'train': dataset, 'test': dataset, 'val': dataset}}
```
"""
splits = config.data.splits

Expand Down
3 changes: 3 additions & 0 deletions federatedscope/nlp/trainer/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@


class NLPTrainer(GeneralTorchTrainer):
"""
``NLPTrainer`` is used for text data.
"""
def _hook_on_batch_forward(self, ctx):
x, label = [move_to(_, ctx.device) for _ in ctx.data_batch]
if isinstance(x, dict):
Expand Down