experiment

AlonEirew · Aug 22, 2020 · 06574c2 · 06574c2
1 parent e80d52b
commit 06574c2
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -13,5 +13,7 @@ src/main/java/workers/ReadCompanyFactory.java
 src/main/java/workers/ReadCompanyWorker.java
 src/main/java/experimentscripts/ExtractCompanyMain.java
 src/main/java/wec/extractors/ExtractCompany.java
+src/test/java/wec/TestExtractCompany.java
 src/main/java/data/CompanyObj.java
 infobox_config/company_info.json
+src/test/resources/wiki_links/company.json
diff --git a/src/main/java/utils/WikipediaUtils.java b/src/main/java/utils/WikipediaUtils.java
@@ -6,13 +6,21 @@
 public class WikipediaUtils {
     public static String cleanTextField(String html) {
         String cleanHtml = html;
-        Pattern pat1 = Pattern.compile("(?s)\\{\\{[^{]*?\\}\\}");
+        Pattern pat1 = Pattern.compile("(?s)\\{\\{.*}}");
+        Pattern pat2 = Pattern.compile("(?s)\\[.*]");
         Matcher match1 = pat1.matcher(cleanHtml);
+
         while (match1.find()) {
             cleanHtml = match1.replaceAll("");
             match1 = pat1.matcher(cleanHtml);
         }
 
+        Matcher match2 = pat2.matcher(cleanHtml);
+        while (match2.find()) {
+            cleanHtml = match2.replaceAll("");
+            match2 = pat2.matcher(cleanHtml);
+        }
+
         return cleanHtml;
     }
 }