diff --git a/.gitignore b/.gitignore
index 0a6ada8e4..79912b364 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,14 @@
target/
.DS_STORE
-.idea/*
-*.iml
.settings/*
javadocs/*
.classpath
.project
logs/
+
+# IntelliJ IDEA
+**/.idea/dictionaries/
+**/.idea/dataSources.*
+**/.idea/libraries/
+**/.idea/tasks.xml
+**/.idea/workspace.xml
diff --git a/.idea/.name b/.idea/.name
new file mode 100644
index 000000000..3dd7124b0
--- /dev/null
+++ b/.idea/.name
@@ -0,0 +1 @@
+crawler4j
\ No newline at end of file
diff --git a/.idea/codeStyleSettings.xml b/.idea/codeStyleSettings.xml
new file mode 100644
index 000000000..3d9da9bb8
--- /dev/null
+++ b/.idea/codeStyleSettings.xml
@@ -0,0 +1,84 @@
+
+
+
+
+
+
+
+
diff --git a/.idea/compiler.xml b/.idea/compiler.xml
new file mode 100644
index 000000000..ce2a4f5ef
--- /dev/null
+++ b/.idea/compiler.xml
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/copyright/profiles_settings.xml b/.idea/copyright/profiles_settings.xml
new file mode 100644
index 000000000..e7bedf337
--- /dev/null
+++ b/.idea/copyright/profiles_settings.xml
@@ -0,0 +1,3 @@
+
+
+
\ No newline at end of file
diff --git a/.idea/encodings.xml b/.idea/encodings.xml
new file mode 100644
index 000000000..b26911bd0
--- /dev/null
+++ b/.idea/encodings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 000000000..f0a4ebbd0
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,1086 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 000000000..3b312839b
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 000000000..348ef06e6
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,370 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ AOP
+
+
+ Abstraction issuesJava
+
+
+ ActionScript specificJavaScript
+
+
+ Assignment issuesJava
+
+
+ Assignment issuesJavaScript
+
+
+ Bitwise operation issuesJava
+
+
+ Bitwise operation issuesJavaScript
+
+
+ CDI(Contexts and Dependency Injection) issues
+
+
+ Class metricsJava
+
+
+ Class structureJava
+
+
+ Cloning issuesJava
+
+
+ Code maturity issuesJava
+
+
+ Code quality toolsJavaScript
+
+
+ Code style issuesJava
+
+
+ Code style issuesJavaScript
+
+
+ CodeSpring CoreSpring
+
+
+ Compiler issuesJava
+
+
+ Concurrency annotation issuesJava
+
+
+ Control flow issuesJava
+
+
+ Control flow issuesJavaScript
+
+
+ Cucumber Java
+
+
+ DOM issuesJavaScript
+
+
+ Data flow issuesJava
+
+
+ Data flow issuesJavaScript
+
+
+ Declaration redundancyJava
+
+
+ Dependency issuesJava
+
+
+ Encapsulation issuesJava
+
+
+ Error handlingJava
+
+
+ Error handlingJavaScript
+
+
+ Faces Model
+
+
+ Finalization issuesJava
+
+
+ GPath inspectionsGroovy
+
+
+ General
+
+
+ GeneralJava
+
+
+ GeneralJavaScript
+
+
+ Google Web Toolkit issues
+
+
+ Groovy
+
+
+ Hibernate Issues
+
+
+ ImportsJava
+
+
+ Inheritance issuesJava
+
+
+ Initialization issuesJava
+
+
+ Internationalization issues
+
+
+ Internationalization issuesJava
+
+
+ J2ME issuesJava
+
+
+ JBoss Seam issues
+
+
+ JPA issues
+
+
+ JSP Inspections
+
+
+ JUnit issues
+
+
+ JUnit issuesJava
+
+
+ Java
+
+
+ Java EE issues
+
+
+ Java language level issuesJava
+
+
+ Java language level migration aidsJava
+
+
+ JavaBeans issuesJava
+
+
+ JavaFX
+
+
+ JavaScript
+
+
+ JavaScript function metricsJavaScript
+
+
+ JavaScript validity issuesJavaScript
+
+
+ Javadoc issuesJava
+
+
+ Language Injection
+
+
+ Logging issuesJava
+
+
+ Memory issuesJava
+
+
+ Method metricsJava
+
+
+ Method signatureScala
+
+
+ Modularization issuesJava
+
+
+ Naming ConventionsGroovy
+
+
+ Naming conventionsJava
+
+
+ Naming conventionsJavaScript
+
+
+ Numeric issuesJava
+
+
+ Packaging issuesJava
+
+
+ Performance issuesJava
+
+
+ Portability issuesJava
+
+
+ Potentially confusing code constructsJavaScript
+
+
+ Probable bugsGroovy
+
+
+ Probable bugsJava
+
+
+ Probable bugsJavaScript
+
+
+ Properties Files
+
+
+ Properties FilesJava
+
+
+ RESTful Web Service
+
+
+ Resource management issuesJava
+
+
+ Scala
+
+
+ Security issuesJava
+
+
+ Serialization issuesJava
+
+
+ Spring
+
+
+ Spring CoreSpring
+
+
+ Struts
+
+
+ Struts 1Struts
+
+
+ StyleGroovy
+
+
+ TestNG
+
+
+ Threading issuesGroovy
+
+
+ Threading issuesJava
+
+
+ Verbose or redundant code constructsJava
+
+
+ Visibility issuesJava
+
+
+ Web Services
+
+
+ WebSocket issues
+
+
+ toString() issuesJava
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 000000000..6ad24d744
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml
new file mode 100644
index 000000000..6df4889b0
--- /dev/null
+++ b/.idea/sqldialects.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 000000000..35eb1ddfb
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/crawler4j.iml b/crawler4j.iml
new file mode 100644
index 000000000..bb2e3bc56
--- /dev/null
+++ b/crawler4j.iml
@@ -0,0 +1,63 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+