simplify code by generating database every time and not caching it

since we set the random seed it is deterministic and very fast to generate. so caching is unnecessary.

Author: jperl

train.py

@@ -61,8 +61,9 @@ def train(params: ModelParams):
 
   optimizer = torch.optim.SGD(model.parameters(), lr=params.lr, momentum=params.momentum)
   loss_fn = torch.nn.BCEWithLogitsLoss()
-  train_loader = DataLoader(XORDataset(), batch_size=params.batch_size, shuffle=True)
-  test_loader = DataLoader(XORDataset(train=False), batch_size=params.batch_size)
+  train_loader = DataLoader(XORDataset(), batch_size=params.batch_size)
+  # test separately generated xor
+  test_loader = DataLoader(XORDataset(), batch_size=params.batch_size)
 
   step = 0
   epoch = 1

utils.py

@@ -16,19 +16,3 @@ def register_parser_types(parser, params_named_tuple):
 
   for key, _type in hints.items():
     parser.add_argument(f'--{key}', type=_type, default=defaults.get(key))
-
-
-# ------------------------- Path Utils -------------------------
-
-
-def ensure_path(path):
-  """Create the path if it does not exist"""
-  if not os.path.exists(path):
-    os.makedirs(path)
-  return path
-
-
-def remove_path(path):
-  """Remove the path if it exists."""
-  if os.path.exists(path):
-    shutil.rmtree(path)

xor_dataset.py

@@ -3,8 +3,6 @@
 import torch
 import torch.utils.data as data
 
-from utils import ensure_path, remove_path
-
 DEFAULT_NUM_BITS = 50
 DEFAULT_NUM_SEQUENCES = 100000
 
@@ -14,41 +12,16 @@
 class XORDataset(data.Dataset):
   data_folder = './data'
 
-  def __init__(self, train=True, test_size=0.2):
-    self._test_size = test_size
-    self.train = train
-
-    # cache dataset so training is deterministic
-    self.ensure_sequences()
-
-    filename = 'train.pt' if self.train else 'test.pt'
-    self.features, self.labels = torch.load(f'{self.data_folder}/{filename}')
+  def __init__(self):
+    self.features, self.labels = get_random_bits_parity()
 
     # expand the dimensions for the lstm
     # [batch, bits] -> [batch, bits, 1]
     self.features = np.expand_dims(self.features, -1)
+
     # [batch, parity] -> [batch, parity, 1]
     self.labels = np.expand_dims(self.labels, -1)
 
-  def ensure_sequences(self):
-    if os.path.exists(self.data_folder):
-      return
-
-    ensure_path(self.data_folder)
-
-    features, labels = get_random_bits_parity()
-
-    test_start = int(len(features) * (1 - self._test_size))
-
-    train_set = (features[:test_start], labels[:test_start])
-    test_set = (features[test_start:], labels[test_start:])
-
-    with open(f'{self.data_folder}/train.pt', 'wb') as file:
-      torch.save(train_set, file)
-
-    with open(f'{self.data_folder}/test.pt', 'wb') as file:
-      torch.save(test_set, file)
-
   def __getitem__(self, index):
     return self.features[index, :], self.labels[index]
 
@@ -73,8 +46,3 @@ def get_random_bits_parity(num_sequences=DEFAULT_NUM_SEQUENCES, num_bits=DEFAULT
   parity = bitsum % 2 != 0
 
   return bit_sequences.astype('float32'), parity.astype('float32')
-
-
-if __name__ == '__main__':
-  remove_path(XORDataset.data_folder)
-  XORDataset(test_size=0.2)