exist_ok on makedirs

ThilinaRajapakse · Jan 29, 2020 · a3d8d84 · a3d8d84
1 parent 36256f1
commit a3d8d84
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 33 deletions.
diff --git a/simpletransformers/classification/classification_model.py b/simpletransformers/classification/classification_model.py
@@ -245,8 +245,7 @@ def train_model(
 
         train_dataset = self.load_and_cache_examples(train_examples, verbose=verbose)
 
-        if not os.path.exists(output_dir):
-            os.makedirs(output_dir)
+        os.makedirs(output_dir, exist_ok = True)
 
         global_step, tr_loss = self.train(
             train_dataset,
@@ -583,8 +582,7 @@ def evaluate(self, eval_df, output_dir, multi_label=False, prefix="", verbose=Tr
             )
         else:
             eval_dataset = self.load_and_cache_examples(eval_examples, evaluate=True, verbose=verbose, silent=silent)
-        if not os.path.exists(eval_output_dir):
-            os.makedirs(eval_output_dir)
+        os.makedirs(eval_output_dir,exist_ok = True)
 
         eval_sampler = SequentialSampler(eval_dataset)
         eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args["eval_batch_size"])
@@ -683,8 +681,7 @@ def load_and_cache_examples(
         else:
             output_mode = "classification"
 
-        if not os.path.isdir(self.args["cache_dir"]):
-            os.makedirs(self.args["cache_dir"])
+        os.makedirs(self.args["cache_dir"], exist_ok=True)
 
         mode = "dev" if evaluate else "train"
         cached_features_file = os.path.join(

diff --git a/simpletransformers/experimental/classification/classification_model.py b/simpletransformers/experimental/classification/classification_model.py
@@ -199,8 +199,7 @@ def train_model(self, train_df, multi_label=False, output_dir=None, show_running
         train_dataset = self.load_and_cache_examples(train_examples)
         global_step, tr_loss = self.train(train_dataset, output_dir, show_running_loss=show_running_loss, eval_df=eval_df)
 
-        if not os.path.exists(output_dir):
-            os.makedirs(output_dir)
+        os.makedirs(output_dir,exist_ok=True)
 
         model_to_save = self.model.module if hasattr(self.model, "module") else self.model
         model_to_save.save_pretrained(output_dir)
@@ -308,8 +307,7 @@ def train(self, train_dataset, output_dir, show_running_loss=True, eval_df=None)
                         # Save model checkpoint
                         output_dir_current = os.path.join(output_dir, "checkpoint-{}".format(global_step))
 
-                        if not os.path.exists(output_dir_current):
-                            os.makedirs(output_dir_current)
+                        os.makedirs(output_dir_current,exist_ok=True)
 
                         # Take care of distributed/parallel training
                         model_to_save = model.module if hasattr(model, "module") else model
@@ -370,8 +368,7 @@ def evaluate(self, eval_df, output_dir, multi_label=False, prefix="", **kwargs):
             eval_examples = [InputExample(i, text, None, label) for i, (text, label) in enumerate(zip(eval_df.iloc[:, 0], eval_df.iloc[:, 1]))]
 
         eval_dataset = self.load_and_cache_examples(eval_examples, evaluate=True)
-        if not os.path.exists(eval_output_dir):
-            os.makedirs(eval_output_dir)
+        os.makedirs(eval_output_dir,exist_ok=True)
 
         eval_sampler = SequentialSampler(eval_dataset)
         eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args["eval_batch_size"])

diff --git a/simpletransformers/ner/ner_model.py b/simpletransformers/ner/ner_model.py
@@ -216,8 +216,7 @@ def train_model(
 
         train_dataset = self.load_and_cache_examples(train_data)
 
-        if not os.path.exists(output_dir):
-            os.makedirs(output_dir)
+        os.makedirs(output_dir, exist_ok=True)
 
         global_step, tr_loss = self.train(
             train_dataset,
@@ -398,7 +397,7 @@ def train(self, train_dataset, output_dir, show_running_loss=True, eval_df=None)
                         )
 
                         if not os.path.exists(output_dir_current):
-                            os.makedirs(output_dir_current)
+                            os.makedirs(output_dir_current, exist_ok = True)
 
                         # Take care of distributed/parallel training
                         model_to_save = (
@@ -422,8 +421,7 @@ def train(self, train_dataset, output_dir, show_running_loss=True, eval_df=None)
                             output_dir, "checkpoint-{}".format(global_step)
                         )
 
-                        if not os.path.exists(output_dir_current):
-                            os.makedirs(output_dir_current)
+                        os.makedirs(output_dir_current, exist_ok = True)
 
                         if args["save_eval_checkpoints"]:
                             model_to_save = (
@@ -459,8 +457,8 @@ def train(self, train_dataset, output_dir, show_running_loss=True, eval_df=None)
 
             if (
                 args["save_model_every_epoch"] or args["evaluate_during_training"]
-            ) and not os.path.exists(output_dir_current):
-                os.makedirs(output_dir_current)
+            ):
+                os.makedirs(output_dir_current,exist_ok=True)
 
             if args["save_model_every_epoch"]:
 
@@ -729,8 +727,7 @@ def load_and_cache_examples(
             ),
         )
 
-        if not os.path.isdir(self.args["cache_dir"]):
-            os.makedirs(self.args["cache_dir"])
+        os.makedirs(self.args["cache_dir"],exist_ok=True)
 
         if os.path.exists(cached_features_file) and (
             (not args["reprocess_input_data"] and not no_cache)

diff --git a/simpletransformers/question_answering/question_answering_model.py b/simpletransformers/question_answering/question_answering_model.py
@@ -146,8 +146,7 @@ def load_and_cache_examples(
         args = self.args
         no_cache = args['no_cache']
 
-        if not os.path.isdir(self.args["cache_dir"]):
-            os.makedirs(self.args["cache_dir"])
+        os.makedirs(self.args["cache_dir"],exist_ok=True)
 
         examples = get_examples(examples, is_training=not evaluate)
 
@@ -280,8 +279,7 @@ def train_model(
 
         train_dataset = self.load_and_cache_examples(train_examples)
 
-        if not os.path.exists(output_dir):
-            os.makedirs(output_dir)
+        os.makedirs(output_dir, exist_ok=True)
 
         global_step, tr_loss = self.train(
             train_dataset,
@@ -462,8 +460,7 @@ def train(self, train_dataset, output_dir, show_running_loss=True, eval_data=Non
                             output_dir, "checkpoint-{}".format(global_step)
                         )
 
-                        if not os.path.exists(output_dir_current):
-                            os.makedirs(output_dir_current)
+                        os.makedirs(output_dir_current,exist_ok=True)
 
                         model_to_save = (
                             model.module if hasattr(model, "module") else model
@@ -486,8 +483,7 @@ def train(self, train_dataset, output_dir, show_running_loss=True, eval_data=Non
                             output_dir, "checkpoint-{}".format(global_step)
                         )
 
-                        if not os.path.exists(output_dir_current):
-                            os.makedirs(output_dir_current)
+                        os.makedirs(output_dir_current,exist_ok=True)
 
                         if args["save_eval_checkpoints"]:
                             model_to_save = (
@@ -523,8 +519,8 @@ def train(self, train_dataset, output_dir, show_running_loss=True, eval_data=Non
 
             if (
                 args["save_model_every_epoch"] or args["evaluate_during_training"]
-            ) and not os.path.exists(output_dir_current):
-                os.makedirs(output_dir_current)
+            ):
+                os.makedirs(output_dir_current,exist_ok=True)
 
             if args["save_model_every_epoch"]:
 
@@ -652,8 +648,7 @@ def evaluate(self, eval_data, output_dir):
                     all_results.append(result)
 
         prefix = "test"
-        if not os.path.isdir(output_dir):
-            os.makedirs(output_dir)
+        os.makedirs(output_dir,exist_ok=True)
 
         output_prediction_file = os.path.join(
             output_dir, "predictions_{}.json".format(prefix)