Prune the migration files for Airflow 3 (apache#41120)

* Prune the migration files for Airflow 3

For a long time now, we have been able to create the metadata
database from the ORM without going through the migration files.
However, the migration files were not pruned because it would
result in users being unable to roll back to a previous version.
There's now an opportunity to prune this and start afresh.
The migration files are getting more significant, and there are
benefits to this pruning. Top in my mind is that the FAB migrations
will no longer be in the airflow migration file, which would help in
removing FAB from the Airflow core.

This change also means we can no longer create a new database from the
migration file. We can only do that through the ORM. I considered creating
the 'current schema' in the migration file, which means having the database
representation in the migration file and helping to create it from scratch.
I dropped the idea because there would be elements of FAB in the migration files again.

Due to this change, the db command option "--use-migration-files" was removed
since this creates the database from the migration file.

* fixup! Prune the migration files for Airflow 3

.github/actions/migration_tests/action.yml

@@ -26,7 +26,7 @@ runs:
       run: >
         breeze shell "airflow db reset --skip-init -y &&
         airflow db migrate --to-revision heads &&
-        airflow db downgrade -r e959f08ac86c -y &&
+        airflow db downgrade -r 22ed7efa9da2 -y &&
         airflow db migrate"
       env:
         COMPOSE_PROJECT_NAME: "docker-compose"
@@ -40,7 +40,7 @@ runs:
       run: >
         breeze shell "airflow db reset -y &&
         airflow db migrate &&
-        airflow db downgrade -r e959f08ac86c -y &&
+        airflow db downgrade -r 22ed7efa9da2 -y &&
         airflow db migrate"
         COMPOSE_PROJECT_NAME: "docker-compose"
       env:

airflow/cli/cli_config.py

@@ -704,12 +704,6 @@ def string_lower_type(val):
     action="store_true",
     default=False,
 )
-ARG_DB_USE_MIGRATION_FILES = Arg(
-    ("-m", "--use-migration-files"),
-    help="Use migration files to perform migration",
-    action="store_true",
-    default=False,
-)
 
 # webserver
 ARG_PORT = Arg(
@@ -1546,7 +1540,7 @@ class GroupCommand(NamedTuple):
         name="reset",
         help="Burn down and rebuild the metadata database",
         func=lazy_load_command("airflow.cli.commands.db_command.resetdb"),
-        args=(ARG_YES, ARG_DB_SKIP_INIT, ARG_DB_USE_MIGRATION_FILES, ARG_VERBOSE),
+        args=(ARG_YES, ARG_DB_SKIP_INIT, ARG_VERBOSE),
     ),
     ActionCommand(
         name="upgrade",
@@ -1566,7 +1560,6 @@ class GroupCommand(NamedTuple):
             ARG_DB_FROM_REVISION,
             ARG_DB_FROM_VERSION,
             ARG_DB_RESERIALIZE_DAGS,
-            ARG_DB_USE_MIGRATION_FILES,
             ARG_VERBOSE,
         ),
         hide=True,
@@ -1590,7 +1583,6 @@ class GroupCommand(NamedTuple):
             ARG_DB_FROM_REVISION,
             ARG_DB_FROM_VERSION,
             ARG_DB_RESERIALIZE_DAGS,
-            ARG_DB_USE_MIGRATION_FILES,
             ARG_VERBOSE,
         ),
     ),

airflow/cli/commands/db_command.py

@@ -63,7 +63,7 @@ def resetdb(args):
     print(f"DB: {settings.engine.url!r}")
     if not (args.yes or input("This will drop existing tables if they exist. Proceed? (y/n)").upper() == "Y"):
         raise SystemExit("Cancelled")
-    db.resetdb(skip_init=args.skip_init, use_migration_files=args.use_migration_files)
+    db.resetdb(skip_init=args.skip_init)
 
 
 def upgradedb(args):
@@ -142,7 +142,6 @@ def migratedb(args):
         from_revision=from_revision,
         show_sql_only=args.show_sql_only,
         reserialize_dags=args.reserialize_dags,
-        use_migration_files=args.use_migration_files,
     )
     if not args.show_sql_only:
         print("Database migrating done!")

airflow/migrations/versions/0151_2_10_0_dag_schedule_dataset_alias_reference.py → airflow/migrations/versions/0001_2_10_0_dag_schedule_dataset_alias_reference.py

@@ -20,7 +20,7 @@
 Add dag_schedule_dataset_alias_reference table.
 
 Revision ID: 22ed7efa9da2
-Revises: 8684e37832e6
+Revises:
 Create Date: 2024-08-05 08:41:47.696495
 
 """
@@ -35,7 +35,7 @@
 
 # revision identifiers, used by Alembic.
 revision = "22ed7efa9da2"
-down_revision = "8684e37832e6"
+down_revision = None
 branch_labels = None
 depends_on = None
 airflow_version = "2.10.0"