This repository has been archived by the owner on Mar 22, 2024. It is now read-only.
add export options #18
Merged
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
NicoRenaud
approved these changes
Nov 25, 2020
| """ | ||
|
|
||
| # Output file | ||
| fname = os.path.join(self.outdir, hdf5) | ||
| self.f5 = h5py.File(fname, 'w') |
There was a problem hiding this comment.
by default this will erase the file fname if it exists which can be dangerous if you rerun an experiment but want to keep the previous results. I would add a check to see if the file exists and if it does change the name with a number. train_data_001.hdf5
There was a problem hiding this comment.
Done ! If train_data_001.hdf5 exists, then I change it to train_data_002.hdf5 and so on.
graphprot/NeuralNet.py
Outdated
| if (save_epoch == 'all') or (epoch == nepoch) : | ||
| self._export_epoch_hdf5(epoch, self.data) | ||
|
|
||
| elif (save_epoch == 'intermediate') and (epoch%5 == 0) : |
There was a problem hiding this comment.
you could add save_every=n as named argument of the function so that we can decide how often we save
There was a problem hiding this comment.
Done, defaults is set to 5
Comment on lines
+547
to
+589
| def _export_epoch_hdf5(self, epoch, data): | ||
| """Export the epoch data to the hdf5 file. | ||
| Export the data of a given epoch in train/valid/test group. | ||
| In each group are stored the predcited values (outputs), | ||
| ground truth (targets) and molecule name (mol). | ||
| Args: | ||
| epoch (int): index of the epoch | ||
| data (dict): data of the epoch | ||
| """ | ||
|
|
||
| # create a group | ||
| grp_name = 'epoch_%04d' % epoch | ||
| grp = self.f5.create_group(grp_name) | ||
|
|
||
| grp.attrs['task'] = self.task | ||
| grp.attrs['target'] = self.target | ||
| grp.attrs['batch_size'] = self.batch_size | ||
|
|
||
| # loop over the pass_type : train/valid/test | ||
| for pass_type, pass_data in data.items(): | ||
|
|
||
| # we don't want to breack the process in case of issue | ||
| try: | ||
|
|
||
| # create subgroup for the pass | ||
| sg = grp.create_group(pass_type) | ||
|
|
||
| # loop over the data : target/output/molname | ||
| for data_name, data_value in pass_data.items(): | ||
|
|
||
| # mol name is a bit different | ||
| # since there are strings | ||
| if data_name == 'mol': | ||
| string_dt = h5py.special_dtype(vlen=str) | ||
| sg.create_dataset( | ||
| data_name, data=data_value, dtype=string_dt) | ||
|
|
||
| # output/target values | ||
| else: | ||
| sg.create_dataset(data_name, data=data_value) | ||
|
|
||
| except TypeError: | ||
| logger.exception("Error in export epoch to hdf5") |
There was a problem hiding this comment.
I took that part from Deeprank;)
NicoRenaud
approved these changes
Nov 25, 2020
| """ | ||
|
|
||
| # Output file | ||
| fname = os.path.join(self.outdir, hdf5) | ||
| self.f5 = h5py.File(fname, 'w') |
Comment on lines
+547
to
+589
| def _export_epoch_hdf5(self, epoch, data): | ||
| """Export the epoch data to the hdf5 file. | ||
| Export the data of a given epoch in train/valid/test group. | ||
| In each group are stored the predcited values (outputs), | ||
| ground truth (targets) and molecule name (mol). | ||
| Args: | ||
| epoch (int): index of the epoch | ||
| data (dict): data of the epoch | ||
| """ | ||
|
|
||
| # create a group | ||
| grp_name = 'epoch_%04d' % epoch | ||
| grp = self.f5.create_group(grp_name) | ||
|
|
||
| grp.attrs['task'] = self.task | ||
| grp.attrs['target'] = self.target | ||
| grp.attrs['batch_size'] = self.batch_size | ||
|
|
||
| # loop over the pass_type : train/valid/test | ||
| for pass_type, pass_data in data.items(): | ||
|
|
||
| # we don't want to breack the process in case of issue | ||
| try: | ||
|
|
||
| # create subgroup for the pass | ||
| sg = grp.create_group(pass_type) | ||
|
|
||
| # loop over the data : target/output/molname | ||
| for data_name, data_value in pass_data.items(): | ||
|
|
||
| # mol name is a bit different | ||
| # since there are strings | ||
| if data_name == 'mol': | ||
| string_dt = h5py.special_dtype(vlen=str) | ||
| sg.create_dataset( | ||
| data_name, data=data_value, dtype=string_dt) | ||
|
|
||
| # output/target values | ||
| else: | ||
| sg.create_dataset(data_name, data=data_value) | ||
|
|
||
| except TypeError: | ||
| logger.exception("Error in export epoch to hdf5") |
Sign up for free
to subscribe to this conversation on GitHub.
Already have an account?
Sign in.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Add different export options:
Export training, evaluation and test data in HDF5 (mol name, prediction, y)
Chose data export frequency (all or intermediate (every 5 epochs for now))
Export the best or the last model