fix: avoid running out of disk space (#558)

by cleaning up after each inner synth run

Author: SurferJeffAtGoogle

autosynth/synth.py

@@ -519,82 +519,93 @@ def _inner_main(temp_dir: str) -> int:
 
     working_repo_path = synthtool_git.clone(f"https://github.com/{args.repository}.git")
 
-    os.chdir(working_repo_path)
+    try:
+        os.chdir(working_repo_path)
 
-    git.configure_git(args.github_user, args.github_email)
+        git.configure_git(args.github_user, args.github_email)
 
-    git.setup_branch(branch)
+        git.setup_branch(branch)
 
-    if args.synth_path:
-        os.chdir(args.synth_path)
-
-    metadata_path = os.path.join(args.metadata_path or "", "synth.metadata")
-
-    flags = autosynth.flags.parse_flags()
-    # Override flags specified in synth.py with flags specified in environment vars.
-    for key in flags.keys():
-        env_value = os.environ.get(key, "")
-        if env_value:
-            flags[key] = False if env_value.lower() == "false" else env_value
-
-    metadata = load_metadata(metadata_path)
-    multiple_commits = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_COMMITS]
-    multiple_prs = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_PRS]
-    if (not multiple_commits and not multiple_prs) or not metadata:
-        if change_pusher.check_if_pr_already_exists(branch):
-            return 0
-
-        synth_log = Synthesizer(
-            metadata_path,
-            args.extra_args,
-            deprecated_execution=args.deprecated_execution,
-        ).synthesize(base_synth_log_path)
-
-        if not has_changes():
-            logger.info("No changes. :)")
-            sys.exit(EXIT_CODE_SKIPPED)
-
-        git.commit_all_changes(pr_title)
-        change_pusher.push_changes(1, branch, pr_title, synth_log)
-        return 1
+        if args.synth_path:
+            os.chdir(args.synth_path)
 
-    else:
-        if not multiple_prs and change_pusher.check_if_pr_already_exists(branch):
-            return 0  # There's already an existing PR
-
-        # Enumerate the versions to loop over.
-        sources = metadata.get("sources", [])
-        source_versions = [
-            git_source.enumerate_versions_for_working_repo(metadata_path, sources)
-        ]
-        # Add supported source version types below:
-        source_versions.extend(
-            git_source.enumerate_versions(sources, pathlib.Path(temp_dir))
-        )
+        metadata_path = os.path.join(args.metadata_path or "", "synth.metadata")
 
-        # Prepare to call synthesize loop.
-        synthesizer = Synthesizer(
-            metadata_path, args.extra_args, args.deprecated_execution, "synth.py",
-        )
-        x = SynthesizeLoopToolbox(
-            source_versions,
-            branch,
-            temp_dir,
-            metadata_path,
-            args.synth_path,
-            base_synth_log_path,
-        )
-        if not multiple_commits:
-            change_pusher = SquashingChangePusher(change_pusher)
+        flags = autosynth.flags.parse_flags()
+        # Override flags specified in synth.py with flags specified in environment vars.
+        for key in flags.keys():
+            env_value = os.environ.get(key, "")
+            if env_value:
+                flags[key] = False if env_value.lower() == "false" else env_value
+
+        metadata = load_metadata(metadata_path)
+        multiple_commits = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_COMMITS]
+        multiple_prs = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_PRS]
+        if (not multiple_commits and not multiple_prs) or not metadata:
+            if change_pusher.check_if_pr_already_exists(branch):
+                return 0
+
+            synth_log = Synthesizer(
+                metadata_path,
+                args.extra_args,
+                deprecated_execution=args.deprecated_execution,
+            ).synthesize(base_synth_log_path)
+
+            if not has_changes():
+                logger.info("No changes. :)")
+                sys.exit(EXIT_CODE_SKIPPED)
 
-        # Call the loop.
-        commit_count = synthesize_loop(x, multiple_prs, change_pusher, synthesizer)
+            git.commit_all_changes(pr_title)
+            change_pusher.push_changes(1, branch, pr_title, synth_log)
+            return 1
 
-        if commit_count == 0:
-            logger.info("No changes. :)")
-            sys.exit(EXIT_CODE_SKIPPED)
+        else:
+            if not multiple_prs and change_pusher.check_if_pr_already_exists(branch):
+                return 0  # There's already an existing PR
+
+            # Enumerate the versions to loop over.
+            sources = metadata.get("sources", [])
+            source_versions = [
+                git_source.enumerate_versions_for_working_repo(metadata_path, sources)
+            ]
+            # Add supported source version types below:
+            source_versions.extend(
+                git_source.enumerate_versions(sources, pathlib.Path(temp_dir))
+            )
+
+            # Prepare to call synthesize loop.
+            synthesizer = Synthesizer(
+                metadata_path, args.extra_args, args.deprecated_execution, "synth.py",
+            )
+            x = SynthesizeLoopToolbox(
+                source_versions,
+                branch,
+                temp_dir,
+                metadata_path,
+                args.synth_path,
+                base_synth_log_path,
+            )
+            if not multiple_commits:
+                change_pusher = SquashingChangePusher(change_pusher)
 
-        return commit_count
+            # Call the loop.
+            commit_count = synthesize_loop(x, multiple_prs, change_pusher, synthesizer)
+
+            if commit_count == 0:
+                logger.info("No changes. :)")
+                sys.exit(EXIT_CODE_SKIPPED)
+
+            return commit_count
+    finally:
+        if args.synth_path:
+            # We're generating code in a mono repo.  The state left behind will
+            # probably be useful for generating the next API.
+            pass
+        else:
+            # We're generating a single API in a single repo, and using a different
+            # repo to generate the next API.  So the next synth will not be able to
+            # use any of this state.  Clean it up to avoid running out of disk space.
+            executor.check_call(["git", "clean", "-fdx"], cwd=working_repo_path)
 
 
 if __name__ == "__main__":