Modify robots

Basantloay · May 16, 2021 · c95139e · c95139e
1 parent 4986e7a
commit c95139e
Show file tree

Hide file tree

Showing 16 changed files with 48 additions and 6,969 deletions.
diff --git a/out/production/Search_Engine/com/company/Crawler/Crawler.class b/out/production/Search_Engine/com/company/Crawler/Crawler.class
diff --git a/robots0.txt b/robots0.txt
diff --git a/robots1.txt b/robots1.txt
diff --git a/robots14.txt b/robots14.txt
diff --git a/robots36.txt b/robots36.txt
diff --git a/robots52.txt b/robots52.txt
diff --git a/robots53.txt b/robots53.txt
diff --git a/robots55.txt b/robots55.txt
diff --git a/robots57.txt b/robots57.txt
diff --git a/robots63.txt b/robots63.txt
diff --git a/robots64.txt b/robots64.txt
diff --git a/robots65.txt b/robots65.txt
diff --git a/robots66.txt b/robots66.txt
diff --git a/robots7.txt b/robots7.txt
diff --git a/robots8.txt b/robots8.txt
diff --git a/src/com/company/Crawler/Crawler.java b/src/com/company/Crawler/Crawler.java
@@ -5,14 +5,11 @@
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 
+import java.io.*;
 import java.nio.file.FileSystems;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.io.File;
-import java.io.FileNotFoundException;  // Import this class to handle errors
-import java.io.FileOutputStream;
-import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.channels.Channels;
@@ -43,40 +40,56 @@ public Crawler(int id, LinkedList<String> seedSet, Set<String> seedSetVisited, V
     }
 
     public boolean robots(String args, Integer num) throws IOException {
-
+        boolean cont=false;boolean find=false;
         URL w = new URL(args + "/robots.txt");
-        //Document doc;
-        try {
-            ReadableByteChannel rbc = Channels.newChannel(w.openStream());
-            FileOutputStream robots = new FileOutputStream("robots" + num.toString() + ".txt");
-            robots.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
-            int j = 0;
-            try {
-                Scanner scannedFile = new Scanner((Readable) robots);
-                while (scannedFile.hasNextLine()) {
-                    String line = scannedFile.nextLine();
-                    System.out.println("\n" + line);
-                    if (line.contains("<!DOCTYPE html>")) {
-                        System.out.println("\nl2naha");
-                        robots.close();
-                        new File("robots" + num.toString() + ".txt").delete();
-                        return false;
+        try(BufferedReader in = new BufferedReader(new InputStreamReader(w.openStream()))) {
+            String line = null;
+            while((line = in.readLine()) != null) {
+                System.out.println(line);
+                if (line.contains("<!DOCTYPE html>")) {
+                    System.out.println("\nl2naha");
+                    return false;
+                }
+                else
+                {
+                    if(line.contains("User-agent"))
+                    {
+                        if (line.contains("*")) {
+                            cont = true;
+                        }
+                        find=true;
+                    }
+                    else if(find && cont)
+                    {
+                        if(line.contains("Disallow"))
+                        {
+                            if(line.contains("*"))
+                            {
+
+                            }
+                                else{
+                            //10 b3d disallow
+                            disallowed.add(args + (line.substring(10)));
+                            System.out.println(args + (line.substring(10)));
+                        }
+                        }
+                        else if(line.contains("Allow"))
+                        {
+                            //7 b3d allow
+                            allowed.add(args+(line.substring(7)));
+                            System.out.println(args+(line.substring(7)));
+                        }
                     }
-
                 }
-                return true;
-            } catch (ClassCastException a) {
-                return false;
-
-            }
-        }
-        catch (IOException x) {
-                return false;
             }
+            return true;
 
-
-
-
+        } catch (FileNotFoundException e ) {
+           return false;
+        }catch( IOException a)
+        {
+            return false;
+        }
     }