Merge branch 'main' into suqi-localtest

Author: hunterhector

.github/workflows/main.yml

@@ -37,8 +37,10 @@ jobs:
         run: |
           python -m pip install --progress-bar off --upgrade pip
           pip install --progress-bar off Django django-guardian
-          pip install --progress-bar off pylint==2.6.0 flake8==3.8.2 mypy==0.790 pytest==5.1.3 black==20.8b1
+          pip install --progress-bar off pylint==2.10.2 flake8==3.8.2
+          pip install --progress-bar off mypy==0.910 pytest==5.1.3 black==21.8b0
           pip install --progress-bar off coverage codecov ddt
+          pip install types-termcolor types-PyYAML
       - name: Install deep learning frameworks
         run: |
           pip install --progress-bar off torch==${{ matrix.torch-version }}
@@ -119,14 +121,11 @@ jobs:
           then
               pip install ${{ matrix.test-details.extra }}
           fi
-      - name: Install Forte
+      - name: Install Forte from source to test the latest version.
         run: |
           git clone https://github.com/asyml/forte.git
           cd forte
           pip install --progress-bar off .
-          cd ..
-          # Remove them to avoid confusing pytest.
-          rm -rf forte
       - name: Start elastic server if test elastic search
         if: ${{ matrix.test-details.dep == 'elastic' }}
         run: |
@@ -168,6 +167,11 @@ jobs:
             do
               pip install "src/"$dep
             done
+      - name: Install Forte from source to use the latest version.
+        run: |
+          git clone https://github.com/asyml/forte.git
+          cd forte
+          pip install --progress-bar off .
       - name: Build Docs
         run: |
           cd docs

.github/workflows/pypi.yml

@@ -12,12 +12,11 @@ on:
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    # Upload to PYPI only on tagged commits.
-    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    # Only run PyPI publish on main repo.
+    if: github.repository == ‘asyml/forte’
     strategy:
       matrix:
         wrapper: [ spacy, allennlp, elastic, faiss, gpt2, huggingface, nltk, stanza, tweepy, vader ]
-
     env:
       python-version: 3.7
     steps:

VERSION

@@ -16,7 +16,7 @@
 import sys
 
 # We should be in the `docs` directory, so the VERSION is one step up.
-with open(os.path.join("..", 'VERSION')) as version_file:
+with open(os.path.join("..", "scripts", "VERSION")) as version_file:
     version = version_file.read().strip()
 
 # from unittest.mock import MagicMock

docs/conf.py

@@ -8,69 +8,71 @@ AllenNLP
 ========
 :hidden:`AllenNLP Processors`
 -----------------------------
-.. autoclass:: forte.allennlp.AllenNLPProcessor
+.. autoclass:: fortex.allennlp.AllenNLPProcessor
     :members:
 
 SpaCy
 =====
 :hidden:`SpaCy Processors`
 ----------------------------
-.. autoclass:: forte.spacy.SpacyProcessor
+.. autoclass:: fortex.spacy.SpacyProcessor
     :members:
 
+.. autoclass:: fortex.spacy.SpacyBatchedProcessor
+    :members:
 
 NLTK
 =====
 :hidden:`NLTK Processors`
 ----------------------------
-.. autoclass:: forte.nltk.NLTKPOSTagger
+.. autoclass:: fortex.nltk.NLTKPOSTagger
     :members:
 
-.. autoclass:: forte.nltk.NLTKSentenceSegmenter
+.. autoclass:: fortex.nltk.NLTKSentenceSegmenter
     :members:
 
-.. autoclass:: forte.nltk.NLTKWordTokenizer
+.. autoclass:: fortex.nltk.NLTKWordTokenizer
     :members:
 
-.. autoclass:: forte.nltk.NLTKLemmatizer
+.. autoclass:: fortex.nltk.NLTKLemmatizer
     :members:
 
-.. autoclass:: forte.nltk.NLTKChunker
+.. autoclass:: fortex.nltk.NLTKChunker
     :members:
 
-.. autoclass:: forte.nltk.NLTKNER
+.. autoclass:: fortex.nltk.NLTKNER
     :members:
 
 Stanza
 ======
 :hidden:`Stanza Processors`
 ----------------------------
-.. autoclass:: forte.stanza.StandfordNLPProcessor
+.. autoclass:: fortex.stanza.StandfordNLPProcessor
     :members:
 
 HuggingFace
 ===========
 :hidden:`HuggingFace Processors`
 --------------------------------
-.. autoclass:: forte.huggingface.ZeroShotClassifier
+.. autoclass:: fortex.huggingface.ZeroShotClassifier
     :members:
-.. autoclass:: forte.huggingface.QuestionAnsweringSingle
+.. autoclass:: fortex.huggingface.QuestionAnsweringSingle
     :members:
-.. autoclass:: forte.huggingface.BERTTokenizer
+.. autoclass:: fortex.huggingface.BERTTokenizer
     :members:
-.. autoclass:: forte.huggingface.BioBERTNERPredictor
+.. autoclass:: fortex.huggingface.BioBERTNERPredictor
     :members:
 
 Twitter
 =======
 :hidden:`Twitter Processors`
 ----------------------------
-.. autoclass:: forte.tweepy.TweetSearchProcessor
+.. autoclass:: fortex.tweepy.TweetSearchProcessor
     :members:
 
 Vader
 =====
 :hidden:`Vader Processors`
 ----------------------------
-.. autoclass:: forte.vader.VaderSentimentProcessor
+.. autoclass:: fortex.vader.VaderSentimentProcessor
     :members:

docs/wrappers.rst

@@ -4,12 +4,12 @@ forte_ir_version: "0.0.1"
 
 components:
 
-  - type: "forte.data.readers.MultiPackSentenceReader"
+  - type: "fortex.data.readers.MultiPackSentenceReader"
     configs:
       input_pack_name: "input_src"
       output_pack_name: "output_tgt"
 
-  - type: "forte.gpt2.text_generation_processor.TextGenerationProcessor"
+  - type: "fortex.gpt2.text_generation_processor.TextGenerationProcessor"
     configs:
       max_decoding_length: 128
       temperature: 0.7
@@ -20,17 +20,16 @@ components:
       batcher:
         batch_size: 64
 
-  - type: "forte.nltk.nltk_processors.NLTKWordTokenizer"
+  - type: "fortex.nltk.nltk_processors.NLTKWordTokenizer"
     configs:
     selector:
       type: "forte.data.selector.NameMatchSelector"
       kwargs:
         select_name: "output_tgt"
 
-  - type: "forte.nltk.nltk_processors.NLTKPOSTagger"
+  - type: "fortex.nltk.nltk_processors.NLTKPOSTagger"
     configs:
     selector:
       type: "forte.data.selector.NameMatchSelector"
       kwargs:
-        select_name: "output_tgt"
-
+        select_name: "output_tgt"

examples/gpt2/sample_multipack_pipeline_gpt.yml

@@ -31,9 +31,9 @@ def test_pipeline_construction(self):
 
         # pylint: disable=line-too-long
         p_names = [
-            'forte.gpt2.text_generation_processor.TextGenerationProcessor',
-            'forte.nltk.nltk_processors.NLTKWordTokenizer',
-            'forte.nltk.nltk_processors.NLTKPOSTagger',
+            'fortex.gpt2.text_generation_processor.TextGenerationProcessor',
+            'fortex.nltk.nltk_processors.NLTKWordTokenizer',
+            'fortex.nltk.nltk_processors.NLTKPOSTagger',
         ]
 
         for proc, p_name in zip(p.components, p_names):

examples/gpt2_test.py

@@ -0,0 +1 @@
+0.1.2

scripts/FORTE_VERSION

@@ -0,0 +1 @@
+0.0.2

scripts/VERSION

@@ -1,4 +1,14 @@
+#!/bin/bash
+
+# Find this folder
+d=`dirname $0`
+
+version=`head -1 ${d}/VERSION`
+forte_version=`head -1 ${d}/FORTE_VERSION`
+echo "Updating versions to "${version}", depends on Forte "${forte_version}
+
 for f in src/*/setup.py; do
     echo "Processing $f"
-    sed -r -i"" "s/version=\"[0-9A-Za-z\.]+\",/version=\"$1\",/1" $f
+    sed -r -i"" "s/version=\"[0-9A-Za-z\.]+\",/version=\"${version}\",/1" $f
+    sed -r -i"" "s/forte==[0-9A-Za-z\.]+\",/forte==${forte_version}\",/1" $f
 done

scripts/update_all_version.sh

@@ -0,0 +1 @@
+from fortex.allennlp.allennlp_processors import AllenNLPProcessor

src/allennlp/forte/allennlp/__init__.py

@@ -26,13 +26,13 @@
     PredicateArgument,
     PredicateMention,
 )
-
 from forte.common import ProcessorConfigError
 from forte.common.configuration import Config
 from forte.common.resources import Resources
 from forte.data.data_pack import DataPack
 from forte.processors.base import PackProcessor
-from forte.allennlp.utils_processor import (
+
+from fortex.allennlp.utils_processor import (
     parse_allennlp_srl_tags,
     parse_allennlp_srl_results,
 )
@@ -150,21 +150,17 @@ def default_configs(cls):
 
         Returns: A dictionary with the default config for this processor.
         """
-        config = super().default_configs()
-        config.update(
-            {
-                "processors": "tokenize,pos,depparse",
-                "tag_formalism": "stanford",
-                "overwrite_entries": False,
-                "allow_parallel_entries": True,
-                "stanford_url": MODEL2URL["stanford"],
-                "srl_url": MODEL2URL["srl"],
-                "universal_url": MODEL2URL["universal"],
-                "cuda_devices": [-1],
-                "infer_batch_size": 0,
-            }
-        )
-        return config
+        return {
+            "processors": "tokenize,pos,depparse",
+            "tag_formalism": "stanford",
+            "overwrite_entries": False,
+            "allow_parallel_entries": True,
+            "stanford_url": MODEL2URL["stanford"],
+            "srl_url": MODEL2URL["srl"],
+            "universal_url": MODEL2URL["universal"],
+            "cuda_devices": [-1],
+            "infer_batch_size": 0,
+        }
 
     def _process(self, input_pack: DataPack):
         # handle existing entries

src/allennlp/fortex/allennlp/__init__.py

@@ -16,18 +16,19 @@
 
 setuptools.setup(
     name="forte.allennlp",
-    version="v0.0.1post2",
+    version="0.0.2",
     url="https://github.com/asyml/forte-wrappers/tree/main/src/allennlp",
     description="Provide Forte implementations of a fantastic collection of "
     "NLP tools.",
     long_description=long_description,
     long_description_content_type="text/markdown",
     license="Apache License Version 2.0",
-    packages=["forte.allennlp"],
+    packages=["fortex.allennlp"],
+    namespace_packages=["fortex"],
     include_package_data=True,
     platforms="any",
     install_requires=[
-        "forte==0.1.1",
+        "forte==0.1.2",
         "more-itertools>=8.0.0",
         "allennlp==1.2.0",
         "allennlp-models==1.2.0",

src/allennlp/forte/allennlp/allennlp_processors.py renamed to src/allennlp/fortex/allennlp/allennlp_processors.py

@@ -20,8 +20,7 @@ This project is part of the [CASL Open Source](http://casl-project.ai/) family.
 
 ### Get Started
 
-- First, install the library along with the desired tools. Let's take AllenNLP
-  as an example:
+- First, install the library along with the elastic search tool:
 
 ```shell
 git clone https://github.com/asyml/forte-wrappers.git

src/allennlp/forte/allennlp/utils_processor.py renamed to src/allennlp/fortex/allennlp/utils_processor.py

@@ -0,0 +1,4 @@
+from fortex.elastic.elastic_search_processor import *
+from fortex.elastic.elastic_indexer import *
+from fortex.elastic.elastic_search_index_processor import *
+from fortex.elastic.elastic_search_query_creator import *

src/allennlp/setup.py

@@ -22,20 +22,21 @@
 
 __all__ = ["ElasticSearchIndexer"]
 
+from forte.common.configurable import Configurable
 from forte.common.configuration import Config
 
 # It seems that elastic search shows a lot of debug message, sometimes can
 # suffocate travis. Trying to depress the logging.
 es_logger.setLevel(logging.INFO)
 
 
-class ElasticSearchIndexer:
+class ElasticSearchIndexer(Configurable):
     r"""Indexer class for `Elasticsearch`."""
 
-    def __init__(self, config: Optional[Union[Dict, Config]] = None):
+    def __init__(self, configs: Optional[Union[Dict, Config]] = None):
         super().__init__()
-        self._config = Config(config, self.default_configs())
-        self.elasticsearch = Elasticsearch(hosts=self._config.hosts)
+        self._configs = self.make_configs(configs)
+        self.elasticsearch = Elasticsearch(hosts=self._configs.hosts)
 
     def index(
         self,
@@ -88,7 +89,7 @@ def add(
                 `https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-refresh.html`
                 for more information on "refresh"
         """
-        index_name = index_name if index_name else self._config.index_name
+        index_name = index_name if index_name else self._configs.index_name
         self.elasticsearch.index(  # pylint: disable=unexpected-keyword-arg
             index=index_name, body=document, refresh=refresh
         )
@@ -171,10 +172,10 @@ def search(
 
     @property
     def hparams(self):
-        return self._config
+        return self._configs
 
-    @staticmethod
-    def default_configs() -> Dict[str, Any]:
+    @classmethod
+    def default_configs(cls) -> Dict[str, Any]:
         r"""Returns a dictionary of default hyperparameters.
 
         .. code-block:: python