Update FindHttpURLs.py and ChangHttpURLsToHttps.py

Author: mstfbl

src/python/tools/ChangeHttpURLsToHttps.py

@@ -2,10 +2,11 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 # -------------------------------------------------------------------------
+
 # Converts all valid HTTP links to HTTPS, where the fed
 # HTTP links are found in Report_AlterableUrls_FindHttpURLs.csv, which
 # is generated by FindHttpURLs.py
-# usage: python3 ChangeHttpURLsToHttps.py [PATH_TO_Report_FindHttpURLs.txt] [PATH_TO_ROOT_OF_NIMBUSML_DIRECTORY]
+# usage: python3 changeHttpURLsToHttps.py [PATH_TO_Report_FindHttpURLs.txt] [PATH_TO_ROOT_OF_NIMBUSML_DIRECTORY]
 # output: Report_ReplaceHttpsURLs.txt
 
 import sys
@@ -14,15 +15,16 @@
 
 def changeUrls(pathToReportCsv, pathToRootDirectory):
     with open(pathToReportCsv, newline='') as csvFile:
-        csv_reader = csv.reader(csvFile, delimiter=',')
+        csv_reader = csv.reader(csvFile, delimiter='\t')
         line_count = 0
         for row in csv_reader:
             if line_count == 0:
                 line_count += 1
             else:
                 #URL: row[0]
                 #relativePath: row[1]
-                absolutePath = pathToRootDirectory[:-1]+row[1]
+                print(row[1])
+                absolutePath = pathToRootDirectory+row[1]
                 fullText = open(absolutePath).read()
                 fullText = fullText.replace(row[0], row[0].replace('http', 'https'))
                 f = open(absolutePath, 'w')
@@ -34,7 +36,7 @@ def changeUrls(pathToReportCsv, pathToRootDirectory):
 
 def main():
     if len(sys.argv) < 3:
-        print("Usage: python3 ChangeHttpURLsToHttps.py [PATH_TO_Report_FindHttpURLs.txt] [PATH_TO_ORIGINAL_NIMBUSML_DIRECTORY]")
+        print("Usage: python3 changeHttpURLsToHttps.py [PATH_TO_Report_FindHttpURLs.txt] [PATH_TO_ORIGINAL_NIMBUSML_DIRECTORY]")
         exit(1)
     changeUrls(sys.argv[1], sys.argv[2])
 

src/python/tools/FindHttpURLs.py

@@ -2,9 +2,10 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 # -------------------------------------------------------------------------
+
 # Finds all HTTP URLs found in the NimbusML repository
 # Converts all valid HTTP links to HTTPS
-# Usage: python3 FindHttpURLs.py [PATH_TO_NimbusML_REPOSITORY]
+# Usage: python3 findHttpURLs.py [PATH_TO_NimbusML_REPOSITORY]
 # Output: Report_AlterableUrls_FindHttpURLs.csv, [Report_NonAlterableUrls_FindHttpURLs.csv, Report_InvalidUrls_FindHttpURLs.csv]
 
 # Required non-standard pip library: urlextract
@@ -13,17 +14,29 @@
 import os
 import requests
 import csv
+import collections
 from urlextract import URLExtract
 
+def addToDictionary(dict, key, value):
+    if key not in dict:
+        dict[key] = [value]
+    else:
+        if value not in dict[key]:
+            dict[key].append(value)
+    return dict
+
 def findHttpUrls(searchRootDirectory):
+    alterableUrlsStore = {}
+    nonAlterableUrlsStore = {}
+    invalidUrlsStore = {}
     extractor = URLExtract()
     lengthOfOriginalRootPath = -1
     for root, _, files in os.walk(searchRootDirectory, onerror=None):
         if lengthOfOriginalRootPath == -1:
              lengthOfOriginalRootPath = len(root)
         for filename in files:
             absoluteFilePath = os.path.join(root, filename)
-            relativeFilePath = '.' + absoluteFilePath[lengthOfOriginalRootPath-1:]
+            relativeFilePath = '.' + absoluteFilePath[lengthOfOriginalRootPath:]
             try:
                 with open(absoluteFilePath, "rb") as f:
                     data = f.read()
@@ -43,47 +56,45 @@ def findHttpUrls(searchRootDirectory):
                                 try:
                                     newRequest = requests.get(changedSelectedUrl)
                                     if newRequest.status_code == 200:
-                                        alterableUrlsStore.append([selectedUrl, relativeFilePath])
+                                        alterableUrlsStore = addToDictionary(alterableUrlsStore, relativeFilePath, selectedUrl)
                                     else:
-                                        nonAlterableUrlsStore.append([selectedUrl, relativeFilePath])
+                                        nonAlterableUrlsStore = addToDictionary(nonAlterableUrlsStore, relativeFilePath, selectedUrl)
                                 except:
-                                    nonAlterableUrlsStore.append([selectedUrl, relativeFilePath])
+                                    nonAlterableUrlsStore = addToDictionary(nonAlterableUrlsStore, relativeFilePath, selectedUrl)
                             else:
-                                invalidUrlsStore.append([selectedUrl, relativeFilePath])
+                                invalidUrlsStore = addToDictionary(invalidUrlsStore, relativeFilePath, selectedUrl)
                         except ConnectionError:
-                            invalidUrlsStore.append([selectedUrl, relativeFilePath])
+                            invalidUrlsStore = addToDictionary(invalidUrlsStore, relativeFilePath, selectedUrl)
             except (IOError, OSError):
                 pass
-    return
+    makeReports(alterableUrlsStore, nonAlterableUrlsStore, invalidUrlsStore)
 
-def makeReports():
-    fieldnames = ['filepath', 'url']
+def makeReports(alterableUrlsStore, nonAlterableUrlsStore, invalidUrlsStore):
     with open('Report_AlterableUrls_FindHttpURLs.csv', mode='w', newline='') as csv_file:
-        writer = csv.writer(csv_file, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
+        writer = csv.writer(csv_file, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL)
         writer.writerow(["url", "relativeFilepath"])
-        for pair in alterableUrlsStore:
-            writer.writerow([pair[0], pair[1]])
+        for fileKey in alterableUrlsStore:
+            for url in alterableUrlsStore[fileKey]:
+                writer.writerow([url, fileKey])
     with open('Report_NonAlterableUrls_FindHttpURLs.csv', mode='w', newline='') as csv_file:
-        writer = csv.writer(csv_file, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
+        writer = csv.writer(csv_file, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL)
         writer.writerow(["url", "relativeFilepath"])
-        for pair in alterableUrlsStore:
-            writer.writerow([pair[0], pair[1]])
+        for fileKey in nonAlterableUrlsStore:
+            for url in nonAlterableUrlsStore[fileKey]:
+                writer.writerow([url, fileKey])
     with open('Report_InvalidUrls_FindHttpURLs.csv', mode='w', newline='') as csv_file:
-        writer = csv.writer(csv_file, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
+        writer = csv.writer(csv_file, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL)
         writer.writerow(["url", "relativeFilepath"])
-        for pair in alterableUrlsStore:
-            writer.writerow([pair[0], pair[1]])
+        for fileKey in invalidUrlsStore:
+            for url in invalidUrlsStore[fileKey]:
+                writer.writerow([url, fileKey])
     return 
 
 def main():
     if len(sys.argv) < 2:
-        print("Usage: python3 FindHttpURLs.py [PATH_TO_NimbusML_REPOSITORY]")
+        print("Usage: python3 findHttpURLs.py [PATH_TO_NimbusML_REPOSITORY]")
         exit(1)
     findHttpUrls(sys.argv[1])
-    makeReports()
-
-alterableUrlsStore = []
-invalidUrlsStore = []
-nonAlterableUrlsStore = []
+    
 if __name__ == "__main__":
     main()