Add Crawler class

Basantloay · May 16, 2021 · 4986e7a · 4986e7a
1 parent 3d0128c
commit 4986e7a
Show file tree

Hide file tree

Showing 19 changed files with 8,467 additions and 0 deletions.
diff --git a/out/production/Search_Engine/META-INF/Search_Engine.kotlin_module b/out/production/Search_Engine/META-INF/Search_Engine.kotlin_module
diff --git a/out/production/Search_Engine/com/company/Crawler/Crawler.class b/out/production/Search_Engine/com/company/Crawler/Crawler.class
diff --git a/out/production/Search_Engine/com/company/jsoup-1.13.1.jar b/out/production/Search_Engine/com/company/jsoup-1.13.1.jar
diff --git a/robots0.txt b/robots0.txt
diff --git a/robots1.txt b/robots1.txt
@@ -0,0 +1,43 @@
+Sitemap: https://www.cnn.com/sitemaps/cnn/index.xml
+Sitemap: https://www.cnn.com/sitemaps/cnn/news.xml
+Sitemap: https://www.cnn.com/sitemaps/sitemap-section.xml
+Sitemap: https://www.cnn.com/sitemaps/sitemap-interactive.xml
+Sitemap: https://www.cnn.com/ampstories/sitemap.xml
+Sitemap: https://edition.cnn.com/sitemaps/news.xml
+User-agent: *
+Allow: /partners/ipad/live-video.json
+Disallow: /*.jsx$
+Disallow: *.jsx$
+Disallow: /*.jsx/
+Disallow: *.jsx?
+Disallow: /ads/
+Disallow: /aol/
+Disallow: /beta/
+Disallow: /browsers/
+Disallow: /cl/
+Disallow: /cnews/
+Disallow: /cnn_adspaces
+Disallow: /cnnbeta/
+Disallow: /cnnintl_adspaces
+Disallow: /development
+Disallow: /editionssi
+Disallow: /help/cnnx.html
+Disallow: /NewsPass
+Disallow: /NOKIA
+Disallow: /partners/
+Disallow: /pipeline/
+Disallow: /pointroll/
+Disallow: /POLLSERVER/
+Disallow: /pr/
+Disallow: /privacy
+Disallow: /PV/
+Disallow: /Quickcast/
+Disallow: /quickcast/
+Disallow: /QUICKNEWS/
+Disallow: /search/
+Disallow: /terms
+Disallow: /test/
+Disallow: /virtual/
+Disallow: /WEB-INF/
+Disallow: /web.projects/
+Disallow: /webview/
diff --git a/robots14.txt b/robots14.txt
diff --git a/robots36.txt b/robots36.txt
diff --git a/robots52.txt b/robots52.txt
diff --git a/robots53.txt b/robots53.txt
diff --git a/robots55.txt b/robots55.txt
@@ -0,0 +1,17 @@
+User-agent: Googlebot
+Disallow:
+
+User-agent: Googlebot-Image
+Disallow:
+
+User-agent: *
+Disallow: /300250/
+Disallow: /300250
+Disallow: /checkout
+Disallow: /orders
+Disallow: /paypal_checkout
+Disallow: /paypal_authorization_callback
+Disallow: /paypal_cancel_callback
+Disallow: /fb-product-feed
+Disallow: /fb-stitcher-feed
+Disallow: /tmp
diff --git a/robots57.txt b/robots57.txt
diff --git a/robots63.txt b/robots63.txt
diff --git a/robots64.txt b/robots64.txt
diff --git a/robots65.txt b/robots65.txt
diff --git a/robots66.txt b/robots66.txt
diff --git a/robots67.txt b/robots67.txt
diff --git a/robots7.txt b/robots7.txt
diff --git a/robots8.txt b/robots8.txt
diff --git a/src/com/company/jsoup-1.13.1.jar b/src/com/company/jsoup-1.13.1.jar