Add fix for bad zip file and logging for build (#1069)

Author: Thomas-Boi

.github/scripts/build_assets/api_handler.py

@@ -2,12 +2,13 @@
 import sys
 import re
 from typing import List
+from io import FileIO
 
 
 # our base url which leads to devicon
 base_url = "https://api.github.com/repos/devicons/devicon/"
 
-def get_merged_pull_reqs_since_last_release(token):
+def get_merged_pull_reqs_since_last_release(token, log_output: FileIO=sys.stdout):
     """
     Get all the merged pull requests since the last release.
     """
@@ -16,9 +17,8 @@ def get_merged_pull_reqs_since_last_release(token):
     found_last_release = False
     page = 1
 
-    print("Getting PRs since last release.")
     while not found_last_release:
-        data = get_merged_pull_reqs(token, page)
+        data = get_merged_pull_reqs(token, page, log_output)
         # assume we don't encounter it during the loop
         last_release_index = 101 
 
@@ -34,7 +34,7 @@ def get_merged_pull_reqs_since_last_release(token):
     return pull_reqs
 
 
-def get_merged_pull_reqs(token, page):
+def get_merged_pull_reqs(token, page, log_output: FileIO=sys.stdout):
     """
     Get the merged pull requests based on page. There are 
     100 results per page. See https://docs.github.com/en/rest/reference/pulls
@@ -53,7 +53,7 @@ def get_merged_pull_reqs(token, page):
         "page": page
     }
 
-    print(f"Querying the GitHub API for requests page #{page}")
+    print(f"Querying the GitHub API for requests page #{page}", file=log_output)
     response = requests.get(url, headers=headers, params=params)
     if not response:
         print(f"Can't query the GitHub API. Status code is {response.status_code}. Message is {response.text}")

.github/scripts/build_assets/filehandler.py

@@ -1,9 +1,10 @@
 import json
-from zipfile import ZipFile
+from zipfile import ZipFile, is_zipfile
 from pathlib import Path
 from typing import List, Union
 import os
 import re
+from io import FileIO
 
 
 def find_new_icons_in_devicon_json(devicon_json_path: str, icomoon_json_path: str):
@@ -133,7 +134,7 @@ def is_alias(font_version: str, aliases: List[dict]):
     return False
 
 
-def extract_files(zip_path: str, extract_path: str, delete=True):
+def extract_files(zip_path: str, extract_path: str, logfile: FileIO, delete=True):
     """
     Extract the style.css and font files from the devicon.zip
     folder. Must call the gulp task "get-icomoon-files"
@@ -144,32 +145,52 @@ def extract_files(zip_path: str, extract_path: str, delete=True):
     will put the extracted files.
     :param delete, whether the function should delete the zip file
     when it's done.
+    :param logfile
     """
-    print("Extracting zipped files...")
-
+    print("Extracting zipped files...", file=logfile)
+    fixBadZipfile(zip_path, logfile)
+    print(f"it's zipped {is_zipfile(zip_path)}", file=logfile)
     icomoon_zip = ZipFile(zip_path)
     target_files = ('selection.json', 'fonts/', 'fonts/devicon.ttf',
                     'fonts/devicon.woff', 'fonts/devicon.eot',
                     'fonts/devicon.svg', "style.css")
     for file in target_files:
         icomoon_zip.extract(file, extract_path)
 
-    print("Files extracted")
+    print("Files extracted", file=logfile)
 
     if delete:
-        print("Deleting devicon zip file...")
+        print("Deleting devicon zip file...", file=logfile)
         icomoon_zip.close()
         os.remove(zip_path)
 
 
-def rename_extracted_files(extract_path: str):
+def fixBadZipfile(zippath: str, logfile: FileIO):  
+    """
+    Fix a bad zipfile (one that causes zipfile.ZipFile to throw a BadZipfile Error).
+    Taken from https://stackoverflow.com/a/11385480/11683637.
+    """
+    f = open(zippath, 'r+b')  
+    data = f.read()  
+    pos = data.find(b'\x50\x4b\x05\x06') # End of central directory signature  
+    if (pos > 0):  
+        # self._log("Trancating file at location " + str(pos + 22)+ ".")  
+        f.seek(pos + 22)   # size of 'ZIP end of central directory record' 
+        f.truncate()  
+    else:
+        print("Zipfile don't need to be fixed", file=logfile)
+
+    f.close() 
+
+
+def rename_extracted_files(extract_path: str, logfile: FileIO):
     """
     Rename the extracted files selection.json and style.css.
     :param extract_path, the location where the function
     can find the extracted files.
     :return: None.
     """
-    print("Renaming files")
+    print("Renaming files", file=logfile)
     old_to_new_list = [
         {
             "old": Path(extract_path, "selection.json"),
@@ -184,7 +205,7 @@ def rename_extracted_files(extract_path: str):
     for dict_ in old_to_new_list:
         os.replace(dict_["old"], dict_["new"])
 
-    print("Files renamed")
+    print("Files renamed", file=logfile)
 
 
 def create_screenshot_folder(dir, screenshot_name: str="screenshots/"):

.github/scripts/build_assets/selenium_runner/BuildSeleniumRunner.py

@@ -27,7 +27,7 @@ def upload_icomoon(self, icomoon_json_path: str):
         :param icomoon_json_path: a path to the iconmoon.json.
         :raises TimeoutException: happens when elements are not found.
         """
-        print("Uploading icomoon.json file...")
+        print("Uploading icomoon.json file...", file=self.log_output)
 
         # find the file input and enter the file path
         import_btn = self.driver.find_element_by_css_selector(
@@ -44,15 +44,15 @@ def upload_icomoon(self, icomoon_json_path: str):
             raise Exception("Cannot find the confirm button when uploading the icomoon.json" \
                   "Ensure that the icomoon.json is in the correct format for Icomoon.io")
 
-        print("JSON file uploaded.")
+        print("JSON file uploaded.", file=self.log_output)
 
     def upload_svgs(self, svgs: List[str], screenshot_folder: str):
         """
         Upload the SVGs provided in svgs. This will upload the 
         :param svgs: a list of svg Paths that we'll upload to icomoon.
         :param screenshot_folder: the name of the screenshot_folder. 
         """
-        print("Uploading SVGs...")
+        print("Uploading SVGs...", file=self.log_output)
 
         import_btn = self.driver.find_element_by_css_selector(
             SeleniumRunner.SET_IMPORT_BUTTON_CSS
@@ -63,7 +63,7 @@ def upload_svgs(self, svgs: List[str], screenshot_folder: str):
         err_messages = []
         for i in range(len(svgs)):
             import_btn.send_keys(svgs[i])
-            print(f"Uploading {svgs[i]}")
+            print(f"Uploading {svgs[i]}", file=self.log_output)
 
             # see if there are stroke messages or replacing icon message
             # there should be none of the second kind
@@ -83,8 +83,9 @@ def upload_svgs(self, svgs: List[str], screenshot_folder: str):
                     raise Exception(f"Unexpected alert found: {alert}")
 
             self.edit_svg()
-            print(f"Finished editing icon.")
+            print(f"Finished editing icon.", file=self.log_output)
 
+        print("Finished uploading all files.", file=self.log_output)
         if err_messages != []:
             message = "BuildSeleniumRunner - Issues found when uploading SVGs:\n"
             raise Exception(message + '\n'.join(err_messages))
@@ -94,9 +95,9 @@ def upload_svgs(self, svgs: List[str], screenshot_folder: str):
         self.switch_toolbar_option(IcomoonOptionState.SELECT)
         self.select_all_icons_in_top_set()
         new_svgs_path = str(Path(screenshot_folder, "new_svgs.png").resolve())
-        self.driver.save_screenshot(new_svgs_path);
+        self.driver.save_screenshot(new_svgs_path)
 
-        print("Finished uploading the svgs...")
+        print("Finished uploading the svgs...", file=self.log_output)
 
     def take_icon_screenshot(self, screenshot_folder: str):
         """
@@ -105,16 +106,19 @@ def take_icon_screenshot(self, screenshot_folder: str):
         :param screenshot_folder: the name of the screenshot_folder. 
         """
         # take pictures
-        print("Taking screenshot of the new icons...")
+        print("Taking screenshot of the new icons...", file=self.log_output)
         self.go_to_generate_font_page()
 
         # take an overall screenshot of the icons that were just added
         # also include the glyph count
         new_icons_path = str(Path(screenshot_folder, "new_icons.png").resolve())
         main_content_xpath = "/html/body/div[4]/div[2]/div/div[1]"
         main_content = self.driver.find_element_by_xpath(main_content_xpath)
+
+        # wait a bit for all the icons to load before we take a pic
+        time.sleep(SeleniumRunner.MED_WAIT_IN_SEC)
         main_content.screenshot(new_icons_path)
-        print("Saved screenshot of the new icons...")
+        print("Saved screenshot of the new icons...", file=self.log_output)
 
     def go_to_generate_font_page(self):
         """
@@ -137,7 +141,7 @@ def download_icomoon_fonts(self, zip_path: Path):
         what the icons look like.
         :param zip_path: the path to the zip file after it's downloaded.
         """
-        print("Downloading Font files...")
+        print("Downloading Font files...", file=self.log_output)
         if self.current_page != IcomoonPage.SELECTION:
             self.go_to_page(IcomoonPage.SELECTION)
 
@@ -149,7 +153,7 @@ def download_icomoon_fonts(self, zip_path: Path):
         )
         download_btn.click()
         if self.wait_for_zip(zip_path):
-            print("Font files downloaded.")
+            print("Font files downloaded.", file=self.log_output)
         else:
             raise TimeoutError(f"Couldn't find {zip_path} after download button was clicked.")
 

.github/scripts/build_assets/selenium_runner/PeekSeleniumRunner.py

@@ -28,7 +28,7 @@ def peek_svgs(self, svgs: List[str], screenshot_folder: str):
         :return an array of svgs with strokes as strings. These show which icon
         contains stroke.
         """
-        print("Peeking SVGs...")
+        print("Peeking SVGs...", file=self.log_output)
 
         import_btn = self.driver.find_element_by_css_selector(
             SeleniumRunner.GENERAL_IMPORT_BUTTON_CSS
@@ -37,13 +37,13 @@ def peek_svgs(self, svgs: List[str], screenshot_folder: str):
         svgs_with_strokes = []
         for i in range(len(svgs)):
             import_btn.send_keys(svgs[i])
-            print(f"Uploaded {svgs[i]}")
+            print(f"Uploaded {svgs[i]}", file=self.log_output)
 
             alert = self.test_for_possible_alert(self.SHORT_WAIT_IN_SEC)
             if alert == None:
                 pass  # all good
             elif alert == IcomoonAlerts.STROKES_GET_IGNORED_WARNING:
-                print(f"- This icon contains strokes: {svgs[i]}")
+                print(f"- This icon contains strokes: {svgs[i]}", file=self.log_output)
                 svg = Path(svgs[i])
                 svgs_with_strokes.append(f"- {svg.name}")
                 self.click_alert_button(self.ALERTS[alert]["buttons"]["DISMISS"])
@@ -57,9 +57,9 @@ def peek_svgs(self, svgs: List[str], screenshot_folder: str):
         new_svgs_path = str(Path(screenshot_folder, "new_svgs.png").resolve())
         icon_set_xpath = "/html/body/div[4]/div[1]/div[2]/div[1]"
         icon_set = self.driver.find_element_by_xpath(icon_set_xpath)
-        icon_set.screenshot(new_svgs_path);
+        icon_set.screenshot(new_svgs_path)
 
-        print("Finished peeking the svgs...")
+        print("Finished peeking the svgs...", file=self.log_output)
         return svgs_with_strokes
 
     def peek_icons(self, screenshot_folder: str, icon_info: dict):
@@ -68,7 +68,7 @@ def peek_icons(self, screenshot_folder: str, icon_info: dict):
         :param screenshot_folder: the name of the screenshot_folder. 
         :param icon_info: a dictionary containing info on an icon. Taken from the devicon.json.
         """
-        print("Begin peeking at the icons...")
+        print("Begin peeking at the icons...", file=self.log_output)
         # ensure all icons in the set is selected.
         self.select_all_icons_in_top_set()
         self.go_to_page(IcomoonPage.GENERATE_FONT)
@@ -119,4 +119,4 @@ def peek_icons(self, screenshot_folder: str, icon_info: dict):
 
             i += 1
 
-        print("Finished peeking the icons...")
+        print("Finished peeking the icons...", file=self.log_output)