From b75904ee36609c8b5653f6bd4a729a03bec71990 Mon Sep 17 00:00:00 2001 From: M Hightower <27247790+mhightower83@users.noreply.github.com> Date: Thu, 24 Mar 2022 16:13:16 -0700 Subject: [PATCH] Fix script failure under windows Rely on argpaser for checking that all arguments are present. Removed redundant argument check in main(). Added '--debug' option and print_dbg method. Rethink failures on overrides. Remove well know path fallbacks, error exit when override file is missing. In well-known path search for preferences.txt, do not assume true. Make failure to find an error exit event. When Windows has two preferences.txt files and they have different values for caching and globals.h is used, error exit. It is not possible to know from the script which is being used. --- tools/mkbuildoptglobals.py | 352 ++++++++++++++++++++++++------------- 1 file changed, 232 insertions(+), 120 deletions(-) mode change 100755 => 100644 tools/mkbuildoptglobals.py diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py old mode 100755 new mode 100644 index 6e3895b88e..687088fbe4 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -78,8 +78,9 @@ globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h commonhfile.fqfn={build.core.path}/CommonHFile.h build.opt.fqfn={build.path}/core/build.opt +mkbuildoptglobals.extra_flags= -recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" +recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 @{build.opt.path} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" """ @@ -198,6 +199,7 @@ err_print_flag = False msg_print_buf = "" +debug_enabled = False # Issues trying to address through buffered printing # 1. Arduino IDE 2.0 RC5 does not show stderr text in color. Text printed does @@ -238,6 +240,13 @@ def print_err(*args, **kwargs): print_msg("***", *args, **kwargs) err_print_flag = True +def print_dbg(*args, **kwargs): + global debug_enabled + global err_print_flag + if debug_enabled: + print_msg("DEBUG:", *args, **kwargs) + err_print_flag = True + def handle_error(err_no): # on err_no 0, commit print buffer to stderr or stdout @@ -405,9 +414,19 @@ def discover_1st_time_run(build_path): def find_preferences_txt(runtime_ide_path): + """ + Check for perferences.txt in well-known locations. Most OSs have two + possibilities. When "portable" is present, it takes priority. Otherwise, the + remaining path wins. However, Windows has two. Depending on the install + source, the APP store or website download, both may appear and create an + ambiguous result. + + Return two item list - Two non "None" items indicate an ambiguous state. + + OS Path list for Arduino IDE 1.6.0 and newer + from: https://www.arduino.cc/en/hacking/preferences + """ platform_name = platform.system() - # OS Path list for Arduino IDE 1.6.0 and newer - # from: https://www.arduino.cc/en/hacking/preferences if "Linux" == platform_name: # Test for portable 1ST # /portable/preferences.txt (when used in portable mode) @@ -415,16 +434,16 @@ def find_preferences_txt(runtime_ide_path): fqfn = os.path.normpath(runtime_ide_path + "/portable/preferences.txt") # Linux - verified with Arduino IDE 1.8.19 if os.path.exists(fqfn): - return fqfn + return [fqfn, None] fqfn = os.path.expanduser("~/.arduino15/preferences.txt") # Linux - verified with Arduino IDE 1.8.18 and 2.0 RC5 64bit and AppImage if os.path.exists(fqfn): - return fqfn + return [fqfn, None] elif "Windows" == platform_name: fqfn = os.path.normpath(runtime_ide_path + "\portable\preferences.txt") # verified on Windows 10 with Arduino IDE 1.8.19 if os.path.exists(fqfn): - return fqfn + return [fqfn, None] # It is never simple. Arduino from the Windows APP store or the download # Windows 8 and up option will save "preferences.txt" in one location. # The downloaded Windows 7 (and up version) will put "preferences.txt" @@ -439,24 +458,27 @@ def find_preferences_txt(runtime_ide_path): print_err("Multiple 'preferences.txt' files found:") print_err(" " + fqfn) print_err(" " + fqfn2) - return fqfn + return [fqfn, None] else: - return fqfn + return [fqfn, fqfn2] elif os.path.exists(fqfn2): - return fqfn2 + return [fqfn2, None] elif "Darwin" == platform_name: # Portable is not compatable with Mac OS X # see https://docs.arduino.cc/software/ide-v1/tutorials/PortableIDE fqfn = os.path.expanduser("~/Library/Arduino15/preferences.txt") # Mac OS X - unverified if os.path.exists(fqfn): - return fqfn + return [fqfn, None] print_err("File preferences.txt not found on " + platform_name) - return "" + return [None, None] def get_preferences_txt(file_fqfn, key): + # Get Key Value, key is allowed to be missing. + # We assume file file_fqfn exists + basename = os.path.basename(file_fqfn) with open(file_fqfn) as file: for line in file: name, value = line.partition("=")[::2] @@ -464,24 +486,42 @@ def get_preferences_txt(file_fqfn, key): val = value.strip().lower() if val != 'true': val = False - print_msg(f" preferences.txt: {key}={val}") + print_msg(f" {basename}: {key}={val}") return val - print_err(" Key " + key + " not found in preferences.txt. Default to true.") + print_err(f" Key '{key}' not found in file {basename}. Default to true.") return True # If we don't find it just assume it is set True def check_preferences_txt(runtime_ide_path, preferences_file): + key = "compiler.cache_core" # return the state of "compiler.cache_core" found in preferences.txt - file_fqfn = preferences_file - if file_fqfn != None and os.path.exists(file_fqfn): - pass - else: + if preferences_file != None: + if os.path.exists(preferences_file): + print_msg(f"Using preferences from '{preferences_file}'") + return get_preferences_txt(preferences_file, key) + else: + print_err(f"Override preferences file '{preferences_file}' not found.") + + elif runtime_ide_path != None: + # For a particular install, search the expected locations for platform.txt + # This should never fail. file_fqfn = find_preferences_txt(runtime_ide_path) - if file_fqfn == "": - return True # cannot find file - assume enabled + if file_fqfn[0] != None: + print_msg(f"Using preferences from '{file_fqfn[0]}'") + val0 = get_preferences_txt(file_fqfn[0], key) + val1 = val0 + if file_fqfn[1] != None: + val1 = get_preferences_txt(file_fqfn[1], key) + if val0 == val1: # We can safely ignore that there were two preferences.txt files + return val0 + else: + print_err(f"Found too many preferences.txt files with different values for '{key}'") + raise UserWarning + else: + # Something is wrong with the installation or our understanding of the installation. + print_err("'preferences.txt' file missing from well known locations.") - print_msg("Using preferences from " + file_fqfn) - return get_preferences_txt(file_fqfn, "compiler.cache_core") + return None def touch(fname, times=None): @@ -497,29 +537,75 @@ def synchronous_touch(globals_h_fqfn, commonhfile_fqfn): with open(commonhfile_fqfn, 'a'): os.utime(commonhfile_fqfn, ns=(ts.st_atime_ns, ts.st_mtime_ns)) + def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): - if args.runtime_ide_version < 10802: + global docs_url + print_dbg(f"runtime_ide_version: {args.runtime_ide_version}") + if args.runtime_ide_version < 10802: # CI also has version 10607 -- and args.runtime_ide_version != 10607: + # Aggresive core caching - not implemented before version 1.8.2 + # Note, Arduino IDE 2.0 rc5 has version 1.6.7 and has aggressive caching. + print_dbg(f"Old version ({args.runtime_ide_version}) of Arduino IDE no aggressive caching option") return False elif args.cache_core != None: print_msg(f"Preferences override, this prebuild script assumes the 'compiler.cache_core' parameter is set to {args.cache_core}") print_msg(f"To change, modify 'mkbuildoptglobals.extra_flags=(--cache_core | --no_cache_core)' in 'platform.local.txt'") return args.cache_core else: + ide_path = None preferences_fqfn = None - if args.preferences_file != None: - preferences_fqfn = args.preferences_file - elif args.preferences_sketch != None: - preferences_fqfn = os.path.normpath( - os.path.join( - os.path.dirname(source_globals_h_fqfn), - args.preferences_sketch)) - elif args.preferences_env != None: - preferences_fqfn = os.getenv(args.preferences_env) - return check_preferences_txt(runtime_ide_path, preferences_fqfn) + if args.preferences_sketch != None: + preferences_fqfn = os.path.join( + os.path.dirname(source_globals_h_fqfn), + os.path.normpath(args.preferences_sketch)) + else: + if args.preferences_file != None: + preferences_fqfn = args.preferences_file + elif args.preferences_env != None: + preferences_fqfn = args.preferences_env + else: + ide_path = runtime_ide_path + + if preferences_fqfn != None: + preferences_fqfn = os.path.normpath(preferences_fqfn) + root = False + if 'Windows' == platform.system(): + if preferences_fqfn[1:2] == ':\\': + root = True + else: + if preferences_fqfn[0] == '/': + root = True + if not root: + if preferences_fqfn[0] != '~': + preferences_fqfn = os.path.join("~", preferences_fqfn) + preferences_fqfn = os.path.expanduser(preferences_fqfn) + print_dbg(f"determine_cache_state: preferences_fqfn: {preferences_fqfn}") + + try: + caching_enabled = check_preferences_txt(ide_path, preferences_fqfn) + except UserWarning: + if os.path.exists(source_globals_h_fqfn): + caching_enabled = None + print_err(f" runtime_ide_version: {args.runtime_ide_version}") + print_err(f" This must be resolved to use '{globals_name}'") + print_err(f" Read more at {docs_url}") + else: + # We can quietly ignore the problem because we are not needed. + caching_enabled = True + + return caching_enabled """ -TODO sort out which of these are viable solutions +TODO + +aggressive caching workaround +========== ======= ========== +The question needs to be asked, is it a good idea? +With all this effort to aid in determining the cache state, it is rendered +usless when arduino command line switches are used that contradict our +settings. + +Sort out which of these are imperfect solutions should stay in Possible options for handling problems caused by: ./arduino --preferences-file other-preferences.txt @@ -548,6 +634,18 @@ def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): """ +def check_env(env): + system = platform.system() + val = os.getenv(env) + if val == None: + if "Linux" == system or "Windows" == system: + raise argparse.ArgumentTypeError(f'Missing environment variable: {env}') + else: + # OS/Library limitation + raise argparse.ArgumentTypeError('Not supported') + return val + + def parse_args(): extra_txt = '''\ Use platform.local.txt 'mkbuildoptglobals.extra_flags=...' to supply override options: @@ -565,13 +663,15 @@ def parse_args(): parser.add_argument('build_opt_fqfn', help="Build FQFN to build.opt") parser.add_argument('source_globals_h_fqfn', help="Source FQFN Sketch.ino.globals.h") parser.add_argument('commonhfile_fqfn', help="Core Source FQFN CommonHFile.h") + parser.add_argument('--debug', action='store_true', required=False, default=False) group = parser.add_mutually_exclusive_group(required=False) group.add_argument('--cache_core', action='store_true', default=None, help='Assume a "compiler.cache_core" value of true') group.add_argument('--no_cache_core', dest='cache_core', action='store_false', help='Assume a "compiler.cache_core" value of false') group.add_argument('--preferences_file', help='Full path to preferences file') group.add_argument('--preferences_sketch', nargs='?', action='store', const="preferences.txt", help='Sketch relative path to preferences file') - if "Linux" == platform.system(): - group.add_argument('--preferences_env', nargs='?', action='store', const="ARDUINO15_PREFERENCES_FILE", help='Use environment variable for path to preferences file') + # Since the docs say most versions of Windows and Linux support the os.getenv method, suppress the help message. + group.add_argument('--preferences_env', nargs='?', action='store', type=check_env, const="ARDUINO15_PREFERENCES_FILE", help=argparse.SUPPRESS) + # ..., help='Use environment variable for path to preferences file') return parser.parse_args() # ref epilog, https://stackoverflow.com/a/50021771 # ref nargs='*'', https://stackoverflow.com/a/4480202 @@ -580,108 +680,120 @@ def parse_args(): def main(): global build_opt_signature global docs_url + global debug_enabled num_include_lines = 1 args = parse_args() + debug_enabled = args.debug runtime_ide_path = os.path.normpath(args.runtime_ide_path) build_path = os.path.normpath(args.build_path) build_opt_fqfn = os.path.normpath(args.build_opt_fqfn) source_globals_h_fqfn = os.path.normpath(args.source_globals_h_fqfn) commonhfile_fqfn = os.path.normpath(args.commonhfile_fqfn) - if commonhfile_fqfn != None and len(commonhfile_fqfn): - globals_name = os.path.basename(source_globals_h_fqfn) - build_path_core, build_opt_name = os.path.split(build_opt_fqfn) - globals_h_fqfn = os.path.join(build_path_core, globals_name) - - first_time = discover_1st_time_run(build_path) + globals_name = os.path.basename(source_globals_h_fqfn) + build_path_core, build_opt_name = os.path.split(build_opt_fqfn) + globals_h_fqfn = os.path.join(build_path_core, globals_name) - use_aggressive_caching_workaround = determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn) + first_time = discover_1st_time_run(build_path) + if first_time: + print_dbg("First run since Arduino IDE started.") - if first_time or \ - not use_aggressive_caching_workaround or \ - not os.path.exists(commonhfile_fqfn): - enable_override(False, commonhfile_fqfn) - - # A future timestamp on commonhfile_fqfn will cause everything to - # rebuild. This occurred during development and may happen after - # changing the system time. - if time.time_ns() < os.stat(commonhfile_fqfn).st_mtime_ns: - touch(commonhfile_fqfn) - print_err(f"Neutralized future timestamp on build file: {commonhfile_fqfn}") - - if not os.path.exists(build_path_core): - os.makedirs(build_path_core) - print_msg("Clean build, created dir " + build_path_core) + use_aggressive_caching_workaround = determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn) + if use_aggressive_caching_workaround == None: + # Specific rrror messages already buffered + handle_error(1) - if os.path.exists(source_globals_h_fqfn): - print_msg("Using global include from " + source_globals_h_fqfn) - - copy_create_build_file(source_globals_h_fqfn, globals_h_fqfn) - - # globals_h_fqfn timestamp was only updated if the source changed. This - # controls the rebuild on change. We can always extract a new build.opt - # w/o triggering a needless rebuild. - embedded_options = extract_create_build_opt_file(globals_h_fqfn, globals_name, build_opt_fqfn) - - if use_aggressive_caching_workaround: - # commonhfile_fqfn encodes the following information - # 1. When touched, it causes a rebuild of core.a - # 2. When file size is non-zero, it indicates we are using the - # aggressive cache workaround. The workaround is set to true - # (active) when we discover a non-zero length global .h file in - # any sketch. The aggressive workaround is cleared on the 1ST - # compile by the Arduino IDE after starting. - # 3. When the timestamp matches the build copy of globals.h - # (globals_h_fqfn), we know one two things: - # * The cached core.a matches up to the current build.opt and - # globals.h. The current sketch owns the cached copy of core.a. - # * globals.h has not changed, and no need to rebuild core.a - # 4. When core.a's timestamp does not match the build copy of - # the global .h file, we only know we need to rebuild core.a, and - # that is enough. - # - # When the sketch build has a "Sketch.ino.globals.h" file in the - # build tree that exactly matches the timestamp of "CommonHFile.h" - # in the platform source tree, it owns the core.a cache copy. If - # not, or "Sketch.ino.globals.h" has changed, rebuild core. - # A non-zero file size for commonhfile_fqfn, means we have seen a - # globals.h file before and workaround is active. - if os.path.getsize(commonhfile_fqfn): - if (os.path.getmtime(globals_h_fqfn) != os.path.getmtime(commonhfile_fqfn)): - # Need to rebuild core.a - # touching commonhfile_fqfn in the source core tree will cause rebuild. - # Looks like touching or writing unrelated files in the source core tree will cause rebuild. - synchronous_touch(globals_h_fqfn, commonhfile_fqfn) - print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") - elif os.path.getsize(globals_h_fqfn): - enable_override(True, commonhfile_fqfn) + if first_time or \ + not use_aggressive_caching_workaround or \ + not os.path.exists(commonhfile_fqfn): + enable_override(False, commonhfile_fqfn) + + # A future timestamp on commonhfile_fqfn will cause everything to + # rebuild. This occurred during development and may happen after + # changing the system time. + if time.time_ns() < os.stat(commonhfile_fqfn).st_mtime_ns: + touch(commonhfile_fqfn) + print_err(f"Neutralized future timestamp on build file: {commonhfile_fqfn}") + + if not os.path.exists(build_path_core): + os.makedirs(build_path_core) + print_msg("Clean build, created dir " + build_path_core) + + if os.path.exists(source_globals_h_fqfn): + print_msg("Using global include from " + source_globals_h_fqfn) + + copy_create_build_file(source_globals_h_fqfn, globals_h_fqfn) + + # globals_h_fqfn timestamp was only updated if the source changed. This + # controls the rebuild on change. We can always extract a new build.opt + # w/o triggering a needless rebuild. + embedded_options = extract_create_build_opt_file(globals_h_fqfn, globals_name, build_opt_fqfn) + + if use_aggressive_caching_workaround: + # commonhfile_fqfn encodes the following information + # 1. When touched, it causes a rebuild of core.a + # 2. When file size is non-zero, it indicates we are using the + # aggressive cache workaround. The workaround is set to true + # (active) when we discover a non-zero length global .h file in + # any sketch. The aggressive workaround is cleared on the 1ST + # compile by the Arduino IDE after starting. + # 3. When the timestamp matches the build copy of globals.h + # (globals_h_fqfn), we know one two things: + # * The cached core.a matches up to the current build.opt and + # globals.h. The current sketch owns the cached copy of core.a. + # * globals.h has not changed, and no need to rebuild core.a + # 4. When core.a's timestamp does not match the build copy of + # the global .h file, we only know we need to rebuild core.a, and + # that is enough. + # + # When the sketch build has a "Sketch.ino.globals.h" file in the + # build tree that exactly matches the timestamp of "CommonHFile.h" + # in the platform source tree, it owns the core.a cache copy. If + # not, or "Sketch.ino.globals.h" has changed, rebuild core. + # A non-zero file size for commonhfile_fqfn, means we have seen a + # globals.h file before and workaround is active. + if debug_enabled: + ts = os.stat(globals_h_fqfn) + print_dbg(f"globals_h_fqfn ns_stamp = {ts.st_mtime_ns}") + print_dbg(f"getmtime(globals_h_fqfn) {os.path.getmtime(globals_h_fqfn)}") + ts = os.stat(commonhfile_fqfn) + print_dbg(f"commonhfile_fqfn ns_stamp = {ts.st_mtime_ns}") + print_dbg(f"getmtime(commonhfile_fqfn) {os.path.getmtime(commonhfile_fqfn)}") + + if os.path.getsize(commonhfile_fqfn): + if (os.path.getmtime(globals_h_fqfn) != os.path.getmtime(commonhfile_fqfn)): + # Need to rebuild core.a + # touching commonhfile_fqfn in the source core tree will cause rebuild. + # Looks like touching or writing unrelated files in the source core tree will cause rebuild. synchronous_touch(globals_h_fqfn, commonhfile_fqfn) print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") - - add_include_line(build_opt_fqfn, commonhfile_fqfn) - add_include_line(build_opt_fqfn, globals_h_fqfn) - - # Provide context help for build option support. - source_build_opt_h_fqfn = os.path.join(os.path.dirname(source_globals_h_fqfn), "build_opt.h") - if os.path.exists(source_build_opt_h_fqfn) and not embedded_options: - print_err("Build options file '" + source_build_opt_h_fqfn + "' not supported.") - print_err(" Add build option content to '" + source_globals_h_fqfn + "'.") - print_err(" Embedd compiler command-line options in a block comment starting with '" + build_opt_signature + "'.") - print_err(" Read more at " + docs_url) - elif os.path.exists(source_globals_h_fqfn): - if not embedded_options: - print_msg("Tip: Embedd compiler command-line options in a block comment starting with '" + build_opt_signature + "'.") - print_msg(" Read more at " + docs_url) + else: + print_dbg(f"Using old cached 'core.a'") + elif os.path.getsize(globals_h_fqfn): + enable_override(True, commonhfile_fqfn) + synchronous_touch(globals_h_fqfn, commonhfile_fqfn) + print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") else: - print_msg("Note: optional global include file '" + source_globals_h_fqfn + "' does not exist.") + print_dbg(f"Workaround not active/needed") + + add_include_line(build_opt_fqfn, commonhfile_fqfn) + add_include_line(build_opt_fqfn, globals_h_fqfn) + + # Provide context help for build option support. + source_build_opt_h_fqfn = os.path.join(os.path.dirname(source_globals_h_fqfn), "build_opt.h") + if os.path.exists(source_build_opt_h_fqfn) and not embedded_options: + print_err("Build options file '" + source_build_opt_h_fqfn + "' not supported.") + print_err(" Add build option content to '" + source_globals_h_fqfn + "'.") + print_err(" Embedd compiler command-line options in a block comment starting with '" + build_opt_signature + "'.") + print_err(" Read more at " + docs_url) + elif os.path.exists(source_globals_h_fqfn): + if not embedded_options: + print_msg("Tip: Embedd compiler command-line options in a block comment starting with '" + build_opt_signature + "'.") print_msg(" Read more at " + docs_url) - else: - print_err(parser.parse_args('-h'.split())) - # print_err("Too few arguments. Add arguments:") - # print_err(" Runtime IDE path, Build path, Build FQFN build.opt, Source FQFN Sketch.ino.globals.h, Core Source FQFN CommonHFile.h") - handle_error(1) + print_msg("Note: optional global include file '" + source_globals_h_fqfn + "' does not exist.") + print_msg(" Read more at " + docs_url) handle_error(0) # commit print buffer