fix test

ci/doc.sh

@@ -1,10 +1,17 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 # build sphinx document under doc/
 mkdir -p doc/_gen
+
 ./doc/gen_rst_py_tool.py ./espnet/bin/*.py > ./doc/_gen/espnet_bin.rst
-# ./utils/gen_rst_py_bin.py ./utils/*.py > ./doc/_gen/utils_py.rst
-# ./utils/gen_rst_sh_bin.py ./utils/*.sh > ./doc/_gen/utils_sh.rst
 
+(
+    cd ./utils
+    ../doc/gen_rst_py_tool.py ./*.py > ../doc/_gen/utils_py.rst
+)
+
+# ./utils/gen_rst_sh_tool.py ./utils/*.sh > ./doc/_gen/utils_sh.rst
 
 travis-sphinx build --source=doc --nowarn

doc/apis/espnet_bin.rst

@@ -1,7 +1,7 @@
-.. _NN tools:
+.. _core tools:
 
-NN tools
-========
+core tools
+==========
 
 ESPnet provides several command-line tools for training and evaluating neural networks (NN) under ``espnet/bin``:
 

doc/apis/utils_py.rst

@@ -5,4 +5,4 @@ python utility tools
 
 ESPnet provides several command-line python tools under ``utils/``
 
-
+.. include:: ../_gen/utils_py.rst

doc/conf.py

@@ -20,6 +20,7 @@
 import sys
 
 sys.path.insert(0, os.path.abspath('../espnet/nets'))
+sys.path.insert(0, os.path.abspath('../utils'))
 
 # -- General configuration ------------------------------------------------
 

doc/gen_rst_py_tool.py

@@ -15,7 +15,8 @@ def __init__(self, path):
         name = name.replace("/", ".")
         self.name = re.sub(r"^[\.]+", "", name)
         self.module = imm.SourceFileLoader(self.name, path).load_module()
-        assert hasattr(self.module, "get_parser"), f"{path} does not have get_parser()"
+        if not hasattr(self.module, "get_parser"):
+            raise ValueError(f"{path} does not have get_parser()")
 
 
 # parser
@@ -28,7 +29,15 @@ def __init__(self, path):
 args = parser.parse_args()
 
 
-modinfo = [ModuleInfo(p) for p in args.src if "__init__.py" not in p]
+modinfo = []
+
+for p in args.src:
+    try:
+        modinfo.append(ModuleInfo(p))
+    except (ValueError, ModuleNotFoundError) as e:
+        import logging
+        logging.warning(f"ignore {p} ({e})")
+
 
 # print refs
 for m in modinfo:

utils/addjson.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 # encoding: utf-8
 
 # Copyright 2018 Nagoya University (Tomoki Hayashi)
@@ -17,6 +17,8 @@
 
 from espnet.utils.cli_utils import get_commandline_args
 
+is_python2 = sys.version_info[0] == 2
+
 
 def get_parser():
     parser = argparse.ArgumentParser(

utils/apply-cmvn.py

@@ -13,8 +13,9 @@
 from espnet.utils.cli_utils import is_scipy_wav_style
 
 
-def main():
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='apply mean-variance normalization to files',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
     parser.add_argument('--verbose', '-V', default=0, type=int,
@@ -58,7 +59,11 @@ def main():
                         help='Read specifier id. e.g. ark:some.ark')
     parser.add_argument('wspecifier', type=str,
                         help='Write specifier id. e.g. ark:some.ark')
-    args = parser.parse_args()
+    return parser
+
+
+def main():
+    args = get_parser().parse_args()
 
     # logging info
     logfmt = "%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s"

utils/average_checkpoints.py

@@ -49,11 +49,15 @@ def main():
         raise ValueError('Incorrect type of backend')
 
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
+def get_parser():
+    parser = argparse.ArgumentParser(description='average models from snapshot')
     parser.add_argument("--snapshots", required=True, type=str, nargs="+")
     parser.add_argument("--out", required=True, type=str)
     parser.add_argument("--num", default=10, type=int)
     parser.add_argument("--backend", default='chainer', type=str)
-    args = parser.parse_args()
+    return parser
+
+
+if __name__ == '__main__':
+    args = get_parser().parse_args()
     main()

utils/change_yaml.py

@@ -5,8 +5,9 @@
 import yaml
 
 
-def main():
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='change specified attributes of a YAML file',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
     parser.add_argument('inyaml', nargs='?')
@@ -15,7 +16,11 @@ def main():
                         help="e.g -a a.b.c=4 -> {'a': {'b': {'c': 4}}}")
     parser.add_argument('-d', '--delete', action='append', default=[],
                         help='e.g -d a -> "a" is removed from the input yaml')
-    args = parser.parse_args()
+    return parser
+
+
+def main():
+    args = get_parser().parse_args()
 
     if args.inyaml is None:
         indict = {}

utils/compute-cmvn-stats.py

@@ -12,7 +12,7 @@
 from espnet.utils.cli_utils import is_scipy_wav_style
 
 
-def main():
+def get_parser():
     parser = argparse.ArgumentParser(
         description='Compute cepstral mean and '
                     'variance normalization statistics'
@@ -40,7 +40,11 @@ def main():
                         help='Read specifier for feats. e.g. ark:some.ark')
     parser.add_argument('wspecifier_or_wxfilename', type=str,
                         help='Write specifier. e.g. ark:some.ark')
-    args = parser.parse_args()
+    return parser
+
+
+def main():
+    args = get_parser().parse_args()
 
     logfmt = "%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s"
     if args.verbose > 0:

utils/concatjson.py

@@ -18,12 +18,17 @@
 is_python2 = sys.version_info[0] == 2
 
 
-if __name__ == '__main__':
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='concatenate json files',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('jsons', type=str, nargs='+',
                         help='json files')
-    args = parser.parse_args()
+    return parser
+
+
+if __name__ == '__main__':
+    args = get_parser().parse_args()
 
     # logging info
     logfmt = '%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s'

utils/dump-pcm.py

@@ -11,8 +11,9 @@
 from espnet.utils.cli_utils import get_commandline_args
 
 
-def main():
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='dump PCM files from a WAV scp file',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('--write-num-frames', type=str,
                         help='Specify wspecifer for utt2num_frames')
@@ -46,6 +47,11 @@ def main():
              '<segment-id> <recording-id> <start-time> <end-time>'
              'e.g. call-861225-A-0050-0065 call-861225-A 5.0 6.5')
     parser.add_argument('wspecifier', type=str, help='Write specifier')
+    return parser
+
+
+def main():
+    parser = get_parser()
     args = parser.parse_args()
 
     logfmt = "%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s"

utils/filt.py

@@ -11,14 +11,19 @@
 is_python2 = sys.version_info[0] == 2
 
 
-def main(args):
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='filter words in a text file',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('--exclude', '-v', dest='exclude', action='store_true',
                         help='exclude filter words')
     parser.add_argument('filt', type=str, help='filter list')
     parser.add_argument('infile', type=str, help='input file')
-    args = parser.parse_args(args)
+    return parser
+
+
+def main(args):
+    args = get_parser().parse_args(args)
     filter_file(args.infile, args.filt, args.exclude)
 
 

utils/get_yaml.py

@@ -4,13 +4,17 @@
 import yaml
 
 
-def main():
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='get a specified attribute from a YAML file',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
     parser.add_argument('inyaml')
     parser.add_argument('attr', help='foo.bar will access yaml.load(inyaml)["foo"]["bar"]')
-    args = parser.parse_args()
+    return parser
+
+
+def main():
+    args = get_parser().parse_args()
     with open(args.inyaml, 'r') as f:
         indict = yaml.load(f, Loader=yaml.Loader)
 

utils/json2text.py

@@ -13,14 +13,19 @@
 from espnet.utils.cli_utils import get_commandline_args
 
 
-if __name__ == '__main__':
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='convert ASR recognized json to text',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('json', type=str, help='json files')
     parser.add_argument('dict', type=str, help='dict')
     parser.add_argument('ref', type=str, help='ref')
     parser.add_argument('hyp', type=str, help='hyp')
-    args = parser.parse_args()
+    return parser
+
+
+if __name__ == '__main__':
+    args = get_parser().parse_args()
 
     # logging info
     logfmt = "%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s"

utils/json2trn.py

@@ -14,15 +14,20 @@
 from espnet.utils.cli_utils import get_commandline_args
 
 
-def main(args):
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='convert a json to a transcription file with a token dictionary',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('json', type=str, help='json files')
     parser.add_argument('dict', type=str, help='dict')
     parser.add_argument('--num-spkrs', type=int, default=1, help='number of speakers')
     parser.add_argument('--refs', type=str, nargs='+', help='ref for all speakers')
     parser.add_argument('--hyps', type=str, nargs='+', help='hyp for all outputs')
-    args = parser.parse_args(args)
+    return parser
+
+
+def main(args):
+    args = get_parser().parse_args(args)
     convert(args.json, args.dict, args.refs, args.hyps, args.num_spkrs)
 
 

utils/mergejson.py

@@ -18,8 +18,9 @@
 is_python2 = sys.version_info[0] == 2
 
 
-if __name__ == '__main__':
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='merge json files',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('--input-jsons', type=str, nargs='+', action='append',
                         default=[], help='Json files for the inputs')
@@ -31,6 +32,11 @@
     parser.add_argument('--verbose', '-V', default=0, type=int,
                         help='Verbose option')
     parser.add_argument('-O', dest='output', type=str, help='Output json file')
+    return parser
+
+
+if __name__ == '__main__':
+    parser = get_parser()
     args = parser.parse_args()
 
     # logging info

utils/scp2json.py

@@ -13,11 +13,17 @@
 
 is_python2 = sys.version_info[0] == 2
 
-if __name__ == '__main__':
+
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='convert scp to json',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('--key', '-k', type=str,
                         help='key')
+    return parser
+
+
+if __name__ == '__main__':
     args = parser.parse_args()
 
     new_line = {}

utils/splitjson.py

@@ -20,14 +20,19 @@
 from espnet.utils.cli_utils import get_commandline_args
 
 
-if __name__ == '__main__':
+def get_parser():
     parser = argparse.ArgumentParser(
+        description='split a json file for parallel processing',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('json', type=str,
                         help='json file')
     parser.add_argument('--parts', '-p', type=int,
                         help='Number of subparts to be prepared', default=0)
-    args = parser.parse_args()
+    return parser
+
+
+if __name__ == '__main__':
+    args = get_parser().parse_args()
 
     # logging info
     logging.basicConfig(level=logging.INFO,