Write directly, use tqdm

tools/aformatter.py

@@ -1,10 +1,12 @@
 #!/usr/bin/python
 
+from io import TextIOWrapper
+from typing import List
 import re
 import json
 import math
 import traceback
-from typing import List
+import tqdm
 
 END_NORMAL_IMAGE = "164"
 END_WHITEOUT_IMAGE = "166"
@@ -302,7 +304,6 @@ def floor_points(entry: dict):
 
 	return entry
 
-
 def validate(entry: dict):
 	"""
 	Validates the entry. Catch errors and tell warnings related to the entry.
@@ -339,16 +340,17 @@ def validate(entry: dict):
 			print(f"{key} of entry {entry['id']} is still invalid! {entry[key]}")
 	return return_status
 
-def per_line_entries(entries: list):
+def per_line_entries(entries: list, file: TextIOWrapper):
 	"""
 	Returns a string of all the entries, with every entry in one line.
 	"""
-	out = "[\n"
-	for entry in entries:
-		if entry:
-			out += json.dumps(entry, ensure_ascii=False) + ",\n"
-	out = out[:-2] + "\n]"
-	return out
+	file.write("[\n")
+	line_temp = ""
+	for entry in tqdm.tqdm(entries):
+		if line_temp:
+			file.write(line_temp + ",\n")
+		line_temp = json.dumps(entry, ensure_ascii=False)
+	file.write(line_temp + "\n]")
 
 def format_all(entry: dict, silent=False):
 	"""
@@ -387,7 +389,7 @@ def print_(*args, **kwargs):
 	return entry
 
 def format_all_entries(entries):
-	for i in range(len(entries)):
+	for i in tqdm.trange(len(entries)):
 		try:
 			entry_formatted = format_all(entries[i], True)
 			validation_status = validate(entries[i])
@@ -399,8 +401,6 @@ def format_all_entries(entries):
 		except Exception:
 			print(f"Exception occured when formatting ID {entries[i]['id']}")
 			print(traceback.format_exc())
-		if not (i % 200):
-			print(f"{i} checked.")
 
 def go(path):
 
@@ -411,10 +411,10 @@ def go(path):
 
 	format_all_entries(entries)
 
-	print(f"{len(entries)} checked. Writing...")
+	print(f"Writing...")
 
 	with open(path, "w", encoding='utf-8', newline='\n') as f2:
-		f2.write(per_line_entries(entries))
+		per_line_entries(entries, f2)
 
 	print("Writing completed. All done.")
 

tools/merge_out.py

@@ -60,7 +60,7 @@
 
 print('Writing...')
 with open('../web/atlas.json', 'w', encoding='utf-8') as atlas_file:
-	atlas_file.write(per_line_entries(atlas_json))
+	per_line_entries(atlas_json, atlas_file)
 
 with open('../data/read-ids.txt', 'a', encoding='utf-8') as read_ids_file:
 	with open('temp-read-ids.txt', 'r+', encoding='utf-8') as read_ids_temp_file:

tools/oneoff/migrate_atlas_format.py

@@ -7,9 +7,12 @@
 - submitted_by removed
 """
 
+from io import TextIOWrapper
 import re
 import json
 
+import tqdm
+
 END_IMAGE = 166
 INIT_CANVAS_RANGE = (1, END_IMAGE)
 EXPANSION_1_RANGE = (56, END_IMAGE)
@@ -73,16 +76,17 @@ def migrate_atlas_format(entry: dict):
 
 	return toreturn
 
-def per_line_entries(entries: list):
+def per_line_entries(entries: list, file: TextIOWrapper):
 	"""
 	Returns a string of all the entries, with every entry in one line.
 	"""
-	out = "[\n"
-	for entry in entries:
-		if entry:
-			out += json.dumps(entry, ensure_ascii=False) + ",\n"
-	out = out[:-2] + "\n]"
-	return out
+	file.write("[\n")
+	line_temp = ""
+	for entry in tqdm.tqdm(entries):
+		if line_temp:
+			file.write(line_temp + ",\n")
+		line_temp = json.dumps(entry, ensure_ascii=False)
+	file.write(line_temp + "\n]")
 
 if __name__ == '__main__':
 
@@ -93,16 +97,14 @@ def go(path):
 		with open(path, "r+", encoding='UTF-8') as f1:
 			entries = json.loads(f1.read())
 
-		for i in range(len(entries)):
+		for i in tqdm.trange(len(entries)):
 			entry_formatted = migrate_atlas_format(entries[i])
 			entries[i] = entry_formatted
-			if not (i % 1000):
-				print(f"{i} checked.")
 
 		print(f"{len(entries)} checked. Writing...")
 
 		with open(path, "w", encoding='utf-8', newline='\n') as f2:
-			f2.write(per_line_entries(entries))
+			per_line_entries(entries, f2)
 
 		print("Writing completed. All done.")
 

tools/requirements.txt

@@ -1 +1,2 @@
-praw
+praw
+tqdm

tools/scale_back.py

@@ -1,10 +1,12 @@
 #!/usr/bin/python
 
+from io import TextIOWrapper
 import json
 import traceback
 import numpy
 from PIL import Image, ImageDraw
 import gc
+import tqdm
 
 """
 # 166 to 164 with reference of 165
@@ -147,16 +149,17 @@ def remove_white(entry: dict):
 
 	return entry
 
-def per_line_entries(entries: list):
+def per_line_entries(entries: list, file: TextIOWrapper):
 	"""
 	Returns a string of all the entries, with every entry in one line.
 	"""
-	out = "[\n"
-	for entry in entries:
-		if entry:
-			out += json.dumps(entry, ensure_ascii=False) + ",\n"
-	out = out[:-2] + "\n]"
-	return out
+	file.write("[\n")
+	line_temp = ""
+	for entry in tqdm.tqdm(entries):
+		if line_temp:
+			file.write(line_temp + ",\n")
+		line_temp = json.dumps(entry, ensure_ascii=False)
+	file.write(line_temp + "\n]")
 
 def format_all(entry: dict, silent=False):
 	def print_(*args, **kwargs):
@@ -168,7 +171,7 @@ def print_(*args, **kwargs):
 	return entry
 
 def scale_back_entries(entries):
-	for i in range(len(entries)):
+	for i in tqdm.trange(len(entries)):
 		try:
 			entry_formatted = format_all(entries[i], True)
 			entries[i] = entry_formatted
@@ -191,7 +194,7 @@ def go(path):
 	print(f"{len(entries)} checked. Writing...")
 
 	with open(path, "w", encoding='utf-8', newline='\n') as f2:
-		f2.write(per_line_entries(entries))
+		per_line_entries(entries, f2)
 
 	print("Writing completed. All done.")