remove LMTask_Deprecated and LMLSTM_Deprecated

pytext/builtin_task.py

@@ -19,7 +19,6 @@
     EnsembleTask,
     IntentSlotTask,
     LMTask,
-    LMTask_Deprecated,
     MaskedLMTask,
     NewBertClassificationTask,
     NewBertPairClassificationTask,
@@ -77,7 +76,6 @@ def register_builtin_tasks():
             EnsembleTask,
             IntentSlotTask,
             LMTask,
-            LMTask_Deprecated,
             MaskedLMTask,
             NewBertClassificationTask,
             NewBertPairClassificationTask,

pytext/config/config_adapter.py

@@ -552,6 +552,69 @@ def flatten_deprecated_ensemble_config(json_config):
     return json_config
 
 
+@register_adapter(from_version=15)
+def remove_lmtask_deprecated(json_config):
+    for section in find_dicts_containing_key(json_config, "LMTask_Deprecated"):
+        task = section.pop("LMTask_Deprecated")
+
+        create_parameter(task, "data.source", {"TSVDataSource": {}})
+        rename_parameter(task, "data_handler.eval_path", "data.source.eval_filename")
+        rename_parameter(task, "data_handler.test_path", "data.source.test_filename")
+        rename_parameter(task, "data_handler.train_path", "data.source.train_filename")
+        create_parameter(task, "data.source.field_names", ["text"])
+
+        rename_parameter(
+            task, "data_handler.append_bos", "model.inputs.tokens.add_bos_token"
+        )
+        rename_parameter(
+            task, "data_handler.append_eos", "model.inputs.tokens.add_eos_token"
+        )
+
+        rename_parameter(
+            task, "features.shared_module_key", "model.embedding.shared_module_key"
+        )
+        rename_parameter(
+            task, "features.word_feat.embed_dim", "model.embedding.embed_dim"
+        )
+
+        create_parameter(task, "data.batcher", {"PoolingBatcher": {}})
+        rename_parameter(
+            task, "data_handler.eval_batch_size", "data.batcher.eval_batch_size"
+        )
+        rename_parameter(
+            task, "data_handler.test_batch_size", "data.batcher.test_batch_size"
+        )
+        rename_parameter(
+            task, "data_handler.train_batch_size", "data.batcher.train_batch_size"
+        )
+
+        rename_parameter(
+            task,
+            "features.word_feat.vocab_size",
+            "model.inputs.tokens.vocab.size_from_data",
+        )
+        rename_parameter(
+            task,
+            "features.word_feat.vocab_from_train_data",
+            "model.inputs.tokens.vocab.build_from_data",
+        )
+
+        rename_parameter(
+            task,
+            "features.word_feat.vocab_file",
+            "model.inputs.tokens.vocab.vocab_files",
+            lambda x: [{"filepath": x}],
+        )
+
+        delete_parameter(task, "data_handler")
+        delete_parameter(task, "features")
+        delete_parameter(task, "featurizer")
+
+        section["LMTask"] = task
+
+    return json_config
+
+
 def upgrade_one_version(json_config):
     current_version = json_config.get("version", 0)
     adapter = ADAPTERS.get(current_version)

pytext/config/pytext_config.py

@@ -141,4 +141,4 @@ class TestConfig(ConfigBase):
     test_out_path: str = ""
 
 
-LATEST_VERSION = 15
+LATEST_VERSION = 16

pytext/config/test/json_config/v15.json

@@ -0,0 +1,91 @@
+[
+  {
+    "original": {
+      "task": {
+        "LMTask_Deprecated": {
+          "features": {
+            "shared_module_key": "SHARED_EMBEDDING",
+            "word_feat": {
+              "vocab_file": "base_dir/SST-2/sst_vocab.txt",
+              "vocab_size": 15000,
+              "vocab_from_train_data": false
+            }
+          },
+          "data_handler": {
+            "LanguageModelDataHandler": {
+              "train_path": "base_dir/wikitext2/train.txt",
+              "eval_path": "base_dir/wikitext2/valid.txt",
+              "test_path": "base_dir/wikitext2/test.txt",
+              "train_batch_size": 16,
+              "eval_batch_size": 16,
+              "test_batch_size": 16
+            }
+          },
+          "model": {
+            "representation": {
+              "shared_module_key": "SHARED_LSTM"
+            },
+            "decoder": {
+              "hidden_dims": [
+                10
+              ]
+            }
+          }
+        }
+      },
+      "version": 15
+    },
+    "adapted": {
+      "task": {
+        "LMTask": {
+          "data": {
+            "source": {
+              "TSVDataSource": {
+                "field_names": ["text"],
+                "train_filename": "base_dir/wikitext2/train.txt",
+                "test_filename": "base_dir/wikitext2/test.txt",
+                "eval_filename": "base_dir/wikitext2/valid.txt"
+              }
+            },
+            "batcher": {
+              "PoolingBatcher": {
+                "eval_batch_size": 16,
+                "test_batch_size": 16,
+                "train_batch_size": 16
+              }
+            }
+          },
+          "model": {
+            "embedding": {
+              "shared_module_key": "SHARED_EMBEDDING"
+            },
+            "inputs": {
+              "tokens": {
+                "vocab": {
+                  "build_from_data": false,
+                  "size_from_data": 15000,
+                  "vocab_files": [
+                    {
+                      "filepath": "base_dir/SST-2/sst_vocab.txt"
+                    }
+                  ]
+                }
+              }
+            },
+            "representation": {
+              "shared_module_key": "SHARED_LSTM"
+            },
+            "decoder": {
+              "hidden_dims": [
+                10
+              ]
+            }
+          }
+        }
+      },
+      "version": 16
+    }
+  }
+]
+
+

pytext/data/__init__.py

@@ -15,7 +15,6 @@
 from .disjoint_multitask_data_handler import DisjointMultitaskDataHandler
 from .doc_classification_data_handler import DocClassificationDataHandler, RawData
 from .joint_data_handler import JointModelDataHandler
-from .language_model_data_handler import LanguageModelDataHandler
 from .query_document_pairwise_ranking_data_handler import (
     QueryDocumentPairwiseRankingDataHandler,
 )
@@ -38,7 +37,6 @@
     "EvalBatchSampler",
     "generator_iterator",
     "JointModelDataHandler",
-    "LanguageModelDataHandler",
     "PoolingBatcher",
     "RandomizedBatchSampler",
     "QueryDocumentPairwiseRankingDataHandler",