#941: #1009: create crawler

.lift.toml

@@ -1,3 +1,3 @@
 build = "mvn"
 
-ignoreRules = [ "Var", "Varifier" ]
+ignoreRules = [ "Var", "Varifier", "MissingSummary", "PATH_TRAVERSAL_IN" ]

url-updater/pom.xml

@@ -77,7 +77,7 @@
 					<archive>
 						<manifest>
 							<addClasspath>true</addClasspath>
-							<mainClass>com.devonfw.tools.ide.url.folderhandling.UrlRepository</mainClass>
+							<mainClass>com.devonfw.tools.ide.url.UpdateInitiator</mainClass>
 						</manifest>
 					</archive>
 				</configuration>

url-updater/src/main/java/com/devonfw/tools/ide/url/UpdateInitiator.java

@@ -0,0 +1,27 @@
+package com.devonfw.tools.ide.url;
+
+import com.devonfw.tools.ide.url.updater.UpdateManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Path;
+
+public class UpdateInitiator {
+	private static final Logger logger = LoggerFactory.getLogger(UpdateInitiator.class.getName());
+
+	public static void main(String[] args) {
+		if (args.length == 0) {
+			logger.error("Error: Missing path to repository as command line argument.");
+			logger.error("Usage: java UpdateInitiator <path_to_repository>");
+			System.exit(1);
+		}
+		String pathToRepo = args[0];
+		Path repoPath = Path.of(pathToRepo);
+		if (!repoPath.toFile().isDirectory()) {
+			logger.error("Error: Provided path is not a valid directory.");
+			System.exit(1);
+		}
+		UpdateManager updateManager = new UpdateManager(repoPath);
+		updateManager.updateAll();
+	}
+}

url-updater/src/main/java/com/devonfw/tools/ide/url/folderhandling/UrlArtifact.java

@@ -1,9 +1,9 @@
 package com.devonfw.tools.ide.url.folderhandling;
 
-import java.nio.file.Path;
-
 import com.devonfw.tools.ide.url.folderhandling.abstractUrlClasses.AbstractUrlArtifact;
 
+import java.nio.file.Path;
+
 /**
  * An {@link UrlArtifact} represents a file or folder in the directory structure of a devonfw-ide urls repository.
  *
@@ -15,13 +15,13 @@
  */
 public interface UrlArtifact {
 
-  /**
-   * @return the {@link Path} to this {@link AbstractUrlArtifact} as folder or file on the disc.
-   */
-  Path getPath();
+	/**
+	 * @return the {@link Path} to this {@link AbstractUrlArtifact} as folder or file on the disc.
+	 */
+	Path getPath();
 
-  /**
-   * @return name the file-name of this {@link AbstractUrlArtifact}.
-   */
-  String getName();
+	/**
+	 * @return name the file-name of this {@link AbstractUrlArtifact}.
+	 */
+	String getName();
 }

url-updater/src/main/java/com/devonfw/tools/ide/url/folderhandling/UrlArtifactWithParent.java

@@ -7,8 +7,8 @@
  */
 public interface UrlArtifactWithParent<P extends UrlFolder<?>> extends UrlArtifact {
 
-  /**
-   * @return the parent {@link UrlFolder} owning this artifact as child.
-   */
-  P getParent();
+	/**
+	 * @return the parent {@link UrlFolder} owning this artifact as child.
+	 */
+	P getParent();
 }

url-updater/src/main/java/com/devonfw/tools/ide/url/folderhandling/UrlDownloadFile.java

@@ -1,5 +1,7 @@
 package com.devonfw.tools.ide.url.folderhandling;
 
+import com.devonfw.tools.ide.url.folderhandling.abstractUrlClasses.AbstractUrlFile;
+
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.IOException;
@@ -9,8 +11,6 @@
 import java.util.HashSet;
 import java.util.Set;
 
-import com.devonfw.tools.ide.url.folderhandling.abstractUrlClasses.AbstractUrlFile;
-
 /**
  * {@link UrlFile} with the download URLs. Its {@link #getName() name} has to follow one of the following conventions:
  * <ul>
@@ -21,97 +21,93 @@
  */
 public class UrlDownloadFile extends AbstractUrlFile {
 
-  private final Set<String> urls = new HashSet<>();
-
-  /**
-   * The constructor.
-   *
-   * @param parent the {@link #getParent() parent folder}.
-   * @param name the {@link #getName() filename}.
-   */
-  public UrlDownloadFile(UrlVersion parent, String name) {
-
-    super(parent, name);
-  }
-
-  /**
-   * @return the number of #getUrl
-   */
-  public int getUrlCount() {
-
-    return this.urls.size();
-  }
-
-  /**
-   * @param url the download URL to add.
-   */
-  public void addUrl(String url) {
-
-    boolean added = this.urls.add(url);
-    if (added) {
-      this.modified = true;
-    }
-  }
-
-  public Set<String> getUrls() {
-
-    return this.urls;
-  }
-
-  /**
-   * @param url the download URL to remove.
-   */
-  public void removeUrl(String url) {
-
-    boolean removed = this.urls.remove(url);
-    if (removed) {
-      this.modified = true;
-    }
-  }
-
-  public Set<Integer> generateUrlHashes() {
-
-    Set<String> urlsSet = getUrls();
-    Set<Integer> urlHashes = new HashSet<>();
-    for (String url : urlsSet) {
-      Integer urlHash = url.hashCode();
-      urlHashes.add(urlHash);
-
-    }
-    return urlHashes;
-
-  }
-
-  @Override
-  public void doLoad() {
-
-    this.urls.clear();
-    Path path = getPath();
-    try (BufferedReader br = Files.newBufferedReader(path)) {
-      String line;
-      do {
-        line = br.readLine();
-        if (line != null) {
-          this.urls.add(line.trim());
-        }
-      } while (line != null);
-    } catch (IOException e) {
-      throw new IllegalStateException("Failed to load file " + path, e);
-    }
-  }
-
-  @Override
-  public void doSave() {
-
-    Path path = getPath();
-    try (BufferedWriter bw = Files.newBufferedWriter(path, StandardOpenOption.TRUNCATE_EXISTING,
-        StandardOpenOption.WRITE, StandardOpenOption.CREATE)) {
-      for (String line : this.urls) {
-        bw.write(line + "\n");
-      }
-    } catch (IOException e) {
-      throw new IllegalStateException("Failed to save file " + path, e);
-    }
-  }
-
+	private final Set<String> urls = new HashSet<>();
+
+	/**
+	 * The constructor.
+	 *
+	 * @param parent the {@link #getParent() parent folder}.
+	 * @param name   the {@link #getName() filename}.
+	 */
+	public UrlDownloadFile(UrlVersion parent, String name) {
+
+		super(parent, name);
+	}
+
+	/**
+	 * @return the number of #getUrl
+	 */
+	public int getUrlCount() {
+
+		return this.urls.size();
+	}
+
+	/**
+	 * @param url the download URL to add.
+	 */
+	public void addUrl(String url) {
+
+		boolean added = this.urls.add(url);
+		if (added) {
+			this.modified = true;
+		}
+	}
+
+
+	public Set<String> getUrls() {
+		return this.urls;
+	}
+
+	/**
+	 * @param url the download URL to remove.
+	 */
+	public void removeUrl(String url) {
+
+		boolean removed = this.urls.remove(url);
+		if (removed) {
+			this.modified = true;
+		}
+	}
+
+
+	public Set<Integer> generateUrlHashes() {
+		Set<Integer> urlHashes = new HashSet<>();
+		for (String url : this.urls) {
+			Integer urlHash = url.hashCode();
+			urlHashes.add(urlHash);
+		}
+		return urlHashes;
+
+	}
+
+	@Override
+	public void doLoad() {
+
+		this.urls.clear();
+		Path path = getPath();
+		try (BufferedReader br = Files.newBufferedReader(path)) {
+			String line;
+			do {
+				line = br.readLine();
+				if (line != null) {
+					this.urls.add(line.trim());
+				}
+			} while (line != null);
+		} catch (IOException e) {
+			throw new IllegalStateException("Failed to load file " + path, e);
+		}
+	}
+
+	@Override
+	public void doSave() {
+
+		Path path = getPath();
+		try (BufferedWriter bw = Files.newBufferedWriter(path, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) {
+			for (String line : this.urls) {
+				bw.write(line + "\n");
+			}
+		} catch (IOException e) {
+			throw new IllegalStateException("Failed to save file " + path, e);
+		}
+	}
 }

url-updater/src/main/java/com/devonfw/tools/ide/url/folderhandling/UrlEdition.java

@@ -9,25 +9,26 @@
  */
 public class UrlEdition extends AbstractUrlFolderWithParent<UrlTool, UrlVersion> {
 
-  /**
-   * The constructor.
-   *
-   * @param parent the {@link #getParent() parent folder}.
-   * @param name the {@link #getName() filename}.
-   */
-  public UrlEdition(UrlTool parent, String name) {
+	/**
+	 * The constructor.
+	 *
+	 * @param parent the {@link #getParent() parent folder}.
+	 * @param name   the {@link #getName() filename}.
+	 */
+	public UrlEdition(UrlTool parent, String name) {
 
-    super(parent, name);
-  }
+		super(parent, name);
+	}
 
-  /**
-   * This method is used to add new children to the children collection of an instance from this class.
-   *
-   * @param name The name of the {@link UrlVersion} object that should be created.
-   */
-  @Override
-  protected UrlVersion newChild(String name) {
+	/**
+	 * This method is used to add new children to the children collection of an instance from this class.
+	 *
+	 * @param name The name of the {@link UrlVersion} object that should be created.
+	 */
+	@Override
+	protected UrlVersion newChild(String name) {
+
+		return new UrlVersion(this, name);
+	}
 
-    return new UrlVersion(this, name);
-  }
 }

url-updater/src/main/java/com/devonfw/tools/ide/url/folderhandling/UrlFolder.java

@@ -2,29 +2,35 @@
 
 import com.devonfw.tools.ide.url.folderhandling.abstractUrlClasses.AbstractUrlArtifact;
 
+import java.util.Collection;
+
 /**
  * Interface for an {@link UrlArtifact} representing a folder containing other {@link UrlArtifact artifacts} as
  * children.
  */
 public interface UrlFolder<C extends UrlArtifactWithParent> extends UrlArtifact {
 
-  /**
-   * @return the (current) number of {@link #getChild(String) children} contained in this folder.
-   */
-  int getChildCount();
+	/**
+	 * @return the (current) number of {@link #getChild(String) children} contained in this folder.
+	 */
+	int getChildCount();
 
-  /**
-   * @param name the {@link #getName() name} of the requested child.
-   * @return the child {@link AbstractUrlArtifact artifact} with the given {@link #getName() name} or {@code null} if
-   *         not found.
-   */
-  C getChild(String name);
+	/**
+	 * @param name the {@link #getName() name} of the requested child.
+	 * @return the child {@link AbstractUrlArtifact artifact} with the given {@link #getName() name} or {@code null} if
+	 * not found.
+	 */
+	C getChild(String name);
 
-  /**
-   * @param name the {@link #getName() name} of the requested child.
-   * @return the child {@link AbstractUrlArtifact artifact} with the given {@link #getName() name}. Will be created if
-   *         it does not already exist.
-   */
-  C getOrCreateChild(String name);
+	/**
+	 * @param name the {@link #getName() name} of the requested child.
+	 * @return the child {@link AbstractUrlArtifact artifact} with the given {@link #getName() name}. Will be created if
+	 * it does not already exist.
+	 */
+	C getOrCreateChild(String name);
 
+	/**
+	 * @return the {@link Collection} of all children of this folder.
+	 */
+	Collection<C> getChildren();
 }