Merge pull request #21 from anetschka/dev

Author: markuskiller

README.md

@@ -14,6 +14,7 @@ This python package is being developed as a `TextBlob` **Language
 Extension**. See [Extension
 Guidelines](https://textblob.readthedocs.org/en/dev/contributing.html)
 for details.
+This repo is my personal fork. It contains German-language adaptations that I consider useful. For the main textblob-de repo, visit [Markus Killer's repo](https://github.com/markuskiller/textblob-de).
 
 Features
 --------
@@ -55,6 +56,10 @@ for details):
     $ pip install -U git+https://github.com/markuskiller/textblob-de.git@dev
     $ python -m textblob.download_corpora
 
+To install this fork, just change the repo URL:
+
+    $ pip install -U git+https://github.com/anetschka/textblob-de.git@dev
+
 Note
 
 `TextBlob` will be installed/upgraded automatically when running

tests/test_blob.py

@@ -293,6 +293,7 @@ def test_translate(self):
         assert_true(translated in ["This is a sentence.",
                                    "This is a sentence .",
                                    "That's a sentence.",
+                                   "That is a sentence.",
                                    "That's a sentence ."])
 
     @expected_failure

tests/test_tokenizers.py

@@ -191,7 +191,7 @@ class TestPatternTokenizer(unittest.TestCase):
     def setUp(self):
         self.tokenizer = PatternTokenizer()
         self.text = "Heute ist der 3. Mai 2014 und Dr. Meier feiert seinen 43. " \
-            "Geburtstag."
+            "Geburtstag. Er wünscht sich z. B. einen Pullover von Frau v. Stein."
         self.snt1 = "Heute ist der 3 ."
 
     def tearDown(self):
@@ -202,8 +202,7 @@ def test_tokenize(self):
                      ['Heute',
                       'ist',
                       'der',
-                      '3',
-                      '.',
+                      '3.',
                       'Mai',
                       '2014',
                       'und',
@@ -214,6 +213,18 @@ def test_tokenize(self):
                       '43',
                       '.',
                       'Geburtstag',
+                      '.',
+                      'Er',
+                      'wünscht',
+                      'sich',
+                      'z.',
+                      'B.',
+                      'einen',
+                      'Pullover',
+                      'von',
+                      'Frau',
+                      'v.',
+                      'Stein',
                       '.'])
 
     def test_exclude_punc(self):
@@ -231,12 +242,23 @@ def test_exclude_punc(self):
                       'feiert',
                       'seinen',
                       '43',
-                      'Geburtstag'])
+                      'Geburtstag',
+                      'Er',
+                      'wünscht',
+                      'sich',
+                      'z',
+                      'B',
+                      'einen',
+                      'Pullover',
+                      'von',
+                      'Frau',
+                      'v',
+                      'Stein'])
 
     def test_tokenize_nested(self):
         assert_equal(self.tokenizer.tokenize(self.text, nested=True),
-                     [['Heute', 'ist', 'der', '3', '.'],
-                      ['Mai',
+                     [['Heute', 'ist', 'der', '3.',
+                      'Mai',
                        '2014',
                        'und',
                        'Dr.',
@@ -245,7 +267,9 @@ def test_tokenize_nested(self):
                        'seinen',
                        '43',
                        '.'],
-                      ['Geburtstag', '.']])
+                      ['Geburtstag', '.'],
+                      ['Er', 'wünscht', 'sich', 'z.', 'B.', 'einen', 'Pullover', 'von', 'Frau', 'v.', 'Stein', '.']
+                      ])
 
     def test_itokenize(self):
         gen = self.tokenizer.itokenize(self.text)
@@ -255,9 +279,10 @@ def test_itokenize(self):
 
     def test_sent_tokenize(self):
         sents = self.tokenizer.sent_tokenize(self.text)
-        assert_equal(sents, ['Heute ist der 3 .',
-                             'Mai 2014 und Dr. Meier feiert seinen 43 .',
-                             'Geburtstag .'])
+        assert_equal(sents, ['Heute ist der 3. Mai 2014 und Dr. Meier feiert seinen 43 .',
+                             'Geburtstag .',
+                             'Er wünscht sich z. B. einen Pullover von Frau v. Stein .'
+                             ])
 
     def test_word_tokenize(self):
         tokens = self.tokenizer.word_tokenize(self.snt1)

textblob_de/ext/_pattern/text/de/__init__.py

@@ -171,16 +171,25 @@ def stts2universal(token, tag):
         return (token, PRON)
     return penntreebank2universal(*stts2penntreebank(token, tag))
 
+#let's add some legal abbreviations, too
+#let's also completely rule out at least simple ordinals
+#let's also rule out anything that could be a date
 ABBREVIATIONS = set((
     "Abs.", "Abt.", "Ass.", "Br.", "Ch.", "Chr.", "Cie.", "Co.", "Dept.", "Diff.", 
     "Dr.", "Eidg.", "Exp.", "Fam.", "Fr.", "Hrsg.", "Inc.", "Inv.", "Jh.", "Jt.", "Kt.", 
     "Mio.", "Mrd.", "Mt.", "Mte.", "Nr.", "Nrn.", "Ord.", "Ph.", "Phil.", "Pkt.", 
     "Prof.", "Pt.", " S.", "St.", "Stv.", "Tit.", "VII.", "al.", "begr.","bzw.", 
-    "chem.", "dent.", "dipl.", "e.g.", "ehem.", "etc.", "excl.", "exkl.", "hum.", 
+    "chem.", "dent.", "dipl.", "e.g.", "ehem.", "etc.", "excl.", "exkl.", "gem.", "hum.", 
     "i.e.", "incl.", "ing.", "inkl.", "int.", "iur.", "lic.", "med.", "no.", "oec.", 
     "phil.", "phys.", "pp.", "psych.", "publ.", "rer.", "sc.", "soz.", "spez.", "stud.", 
-    "theol.", "usw.", "vet.", "vgl.", "vol.", "wiss.",
-    "d.h.", "h.c.", u"o.ä.", "u.a.", "z.B.", "z.T.", "z.Zt."
+    "theol.", "usw.", "v.", "vet.", "vgl.", "vol.", "wiss.",
+    "d.h.", "h.c.", u"o.ä.", "u.a.", "z.B.", "z.T.", "z.Zt.", "z. B.", "d. h.", "h. c.", 
+    u"o. ä.", "u. a.", "z. B.", "z. T.", "z. Zt.",
+    "BGBl.", "ABl.", "Bundesgesetzbl.",
+    "0.", "1.", "2.", "3.", "4.", "5.", "6.", "7.", "8.", "9.", "10.", "11.", "12.", "13.",
+    "14.", "15.", "16.", "17.", "18.", "19.", "20.", "21.", "22.", "23.", "24.", "25.", "26.",
+    "27.", "28.", "29.", "30.", "31."
+    
 ))
 
 def find_lemmata(tokens):