Improve bom generation

allspice/utils/bom_generation.py

@@ -248,31 +248,39 @@ def _extract_all_schdoc_components(
     """
 
     files_in_repo = repository.get_git_content(ref=ref)
-    all_components = []
-    for repo_file in files_in_repo:
-        if repo_file.name in schdoc_files_in_proj:
-            # If the file is not yet generated, we'll retry a few times.
-            retry_count = 0
-            while True:
-                retry_count += 1
-
-                try:
-                    schdoc_file_json = repository.get_generated_json(repo_file, ref=ref)
-                    all_components.extend(
-                        _extract_components_from_schdoc(
-                            schdoc_file_json,
-                            attributes_mapping,
-                        )
-                    )
 
-                    break
-                except NotYetGeneratedException:
-                    if retry_count > 20:
-                        break
+    schdoc_files_in_repo = [file for file in files_in_repo if file.name in schdoc_files_in_proj]
+
+    # Create a retry value for each file so that we can traverse them
+    # in parallel for improved performance
+    retry_counter = [60]*len(schdoc_files_in_repo)
 
-                    # Wait a bit before retrying.
-                    time.sleep(0.5)
-                    continue
+    all_components = []
+    # If the file is not yet generated, we'll retry a few times.
+    while any(i > 0 for i in retry_counter):
+        for idx in range(len(retry_counter)):
+            if retry_counter[idx] < 0:
+                continue
+            elif retry_counter[idx] == 0:
+                raise Exception(
+                    f"Fetching file {schdoc_files_in_repo[idx].path} in {repository} ref={ref} exceeded max retries.")
+
+            retry_counter[idx] -= 1
+
+            try:
+                schdoc_file_json = repository.get_generated_json(schdoc_files_in_repo[idx], ref=ref)
+                all_components.extend(
+                    _extract_components_from_schdoc(
+                        schdoc_file_json,
+                        attributes_mapping,
+                    )
+                )
+                retry_counter[idx] = -1
+            except NotYetGeneratedException:
+                pass
+            if any(retry_counter):
+                # Wait a bit before retrying.
+                time.sleep(0.25)
 
     return all_components
 

allspice/utils/core.py

@@ -17,16 +17,13 @@ def get_all_pcb_components(
     """
 
     retry_count = 0
-    while True:
+    while retry_count < 60:
         retry_count += 1
         try:
             pcb_json = repository.get_generated_json(pcb_file, ref=ref)
-            break
+            return pcb_json["component_instances"]
         except NotYetGeneratedException:
-            if retry_count > 20:
-                break
             # Wait a bit before retrying.
-            time.sleep(0.5)
+            time.sleep(0.25)
             continue
-
-    return pcb_json["component_instances"]
+    raise Exception(f"Fetching file {pcb_file} in {repository} ref={ref} exceeded max retries.")

examples/generate_bom/compute_cogs.py

@@ -187,9 +187,9 @@ def fetch_price_for_part(part_number: str) -> dict[int, float]:
     with ExitStack() as stack:
         if args.output_file:
             file = stack.enter_context(open(args.output_file, "w"))
-            writer = csv.writer(file)
+            writer = csv.writer(file, lineterminator="\n")
         else:
-            writer = csv.writer(sys.stdout)
+            writer = csv.writer(sys.stdout, lineterminator="\n")
 
         writer.writerow(headers)
         writer.writerows(rows)

examples/generate_bom/generate_bom.py

@@ -11,13 +11,20 @@
 from contextlib import ExitStack
 
 from allspice import AllSpice
-from allspice.utils.bom_generation import AttributesMapping, generate_bom_for_altium
+from allspice.utils.bom_generation import AttributesMapping, BomEntry, generate_bom_for_altium
 
 
 with open("attributes_mapping.json", "r") as f:
     attributes_mapper = AttributesMapping.from_dict(json.loads(f.read()))
 
 
+def bom_entry_sort_key(x: BomEntry) -> str:
+    ''' sort BOM list by this criteria'''
+    if len(x.designators):
+        return sorted(x.designators)[0]
+    return "_" + x.part_number
+
+
 if __name__ == "__main__":
     # Parse command line arguments. If you're writing a special purpose script,
     # you can hardcode these values instead of using command line arguments.
@@ -84,20 +91,21 @@
     bom_rows = [
         [
             bom_row.description,
-            ", ".join(bom_row.designators),
+            ", ".join(sorted(bom_row.designators)),
             bom_row.quantity,
             bom_row.manufacturer,
             bom_row.part_number,
         ]
-        for bom_row in bom_rows
+        # Sort by designator
+        for bom_row in sorted(bom_rows, key=bom_entry_sort_key)
     ]
 
     with ExitStack() as stack:
         if args.output_file is not None:
             f = stack.enter_context(open(args.output_file, "w"))
-            writer = csv.writer(f)
+            writer = csv.writer(f, lineterminator="\n")
         else:
-            writer = csv.writer(sys.stdout)
+            writer = csv.writer(sys.stdout, lineterminator="\n")
 
         header = [
             "Description",