chore: Improve Alembic Migration Generation (#4192)

alembic/env.py

@@ -1,4 +1,6 @@
-from sqlalchemy import engine_from_config, pool
+from typing import Any
+
+import sqlalchemy as sa
 
 import mealie.db.models._all_models  # noqa: F401
 from alembic import context
@@ -29,6 +31,28 @@ if not settings.DB_URL:
 config.set_main_option("sqlalchemy.url", settings.DB_URL.replace("%", "%%"))
 
 
+def include_object(object: Any, name: str, type_: str, reflected: bool, compare_to: Any):
+    # skip dropping food/unit unique constraints; they are defined manually so alembic doesn't see them
+    # see: revision dded3119c1fe
+    if type_ == "unique_constraint" and name == "ingredient_foods_name_group_id_key" and compare_to is None:
+        return False
+    if type_ == "unique_constraint" and name == "ingredient_units_name_group_id_key" and compare_to is None:
+        return False
+
+    # skip changing the quantity column in recipes_ingredients; it's a float on postgres, but an integer on sqlite
+    # see: revision 263dd6707191
+    if (
+        type_ == "column"
+        and name == "quantity"
+        and object.table.name == "recipes_ingredients"
+        and hasattr(compare_to, "type")
+        and isinstance(compare_to.type, sa.Integer)
+    ):
+        return False
+
+    return True
+
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -60,15 +84,19 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
-    connectable = engine_from_config(
+    connectable = sa.engine_from_config(
         config.get_section(config.config_ini_section),
         prefix="sqlalchemy.",
-        poolclass=pool.NullPool,
+        poolclass=sa.pool.NullPool,
     )
 
     with connectable.connect() as connection:
         context.configure(
-            connection=connection, target_metadata=target_metadata, user_module_prefix="mealie.db.migration_types."
+            connection=connection,
+            target_metadata=target_metadata,
+            user_module_prefix="mealie.db.migration_types.",
+            render_as_batch=True,
+            include_object=include_object,
         )
 
         with context.begin_transaction():

alembic/script.py.mako

@@ -9,13 +9,15 @@ import sqlalchemy as sa
 
 import mealie.db.migration_types
 from alembic import op
-${imports if imports else ""}
+% if imports:
+${imports}
+% endif
 
 # revision identifiers, used by Alembic.
 revision = ${repr(up_revision)}
-down_revision = ${repr(down_revision)}
+down_revision: str | None = ${repr(down_revision)}
-branch_labels = ${repr(branch_labels)}
+branch_labels: str | tuple[str, ...] | None = ${repr(branch_labels)}
-depends_on = ${repr(depends_on)}
+depends_on: str | tuple[str, ...] | None = ${repr(depends_on)}
 
 
 def upgrade():

alembic/versions/2022-02-21-19.56.24_6b0f5f32d602_initial_tables.py

@@ -14,9 +14,9 @@ from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = "6b0f5f32d602"
-down_revision = None
+down_revision: str | None = None
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 # Adapted from https://improveandrepeat.com/2021/09/python-friday-87-handling-pre-existing-tables-with-alembic-and-sqlalchemy/

alembic/versions/2022-03-23-17.43.34_263dd6707191_convert_quantity_from_integer_to_float.py

@@ -13,8 +13,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "263dd6707191"
 down_revision = "6b0f5f32d602"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def is_postgres():

alembic/versions/2022-03-27-19.30.28_f1a2dbee5fe9_add_original_text_column_to_recipes_.py

@@ -13,8 +13,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "f1a2dbee5fe9"
 down_revision = "263dd6707191"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2022-03-31-19.19.55_59eb59135381_add_tags_to_cookbooks.py

@@ -14,8 +14,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "59eb59135381"
 down_revision = "f1a2dbee5fe9"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2022-04-03-10.48.51_09dfc897ad62_add_require_all_for_cookbook_filters.py

@@ -14,8 +14,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "09dfc897ad62"
 down_revision = "59eb59135381"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2022-06-01-11.12.06_ab0bae02578f_add_use_abbreviation_column_to_.py

@@ -13,8 +13,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "ab0bae02578f"
 down_revision = "09dfc897ad62"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2022-06-15-21.05.34_f30cf048c228_add_new_webhook_fields.py

@@ -14,8 +14,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "f30cf048c228"
 down_revision = "ab0bae02578f"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2022-08-12-19.05.59_188374910655_add_login_attemps_and_locked_at_field_.py

@@ -13,8 +13,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "188374910655"
 down_revision = "f30cf048c228"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2022-08-13-17.07.07_089bfa50d0ed_add_is_ocr_recipe_column_to_recipes.py

@@ -13,8 +13,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "089bfa50d0ed"
 down_revision = "188374910655"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2022-08-29-13.57.40_44e8d670719d_add_extras_to_shopping_lists_list_items_.py

@@ -14,8 +14,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "44e8d670719d"
 down_revision = "089bfa50d0ed"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2022-09-27-14.53.14_2ea7a807915c_add_recipe_timeline_events_table.py

@@ -14,8 +14,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "2ea7a807915c"
 down_revision = "44e8d670719d"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2022-11-03-13.10.24_1923519381ad_renamed_timeline_event_message_and_.py

@@ -13,8 +13,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "1923519381ad"
 down_revision = "2ea7a807915c"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2022-11-22-03.42.45_167eb69066ad_add_recipe_scale_to_shopping_list_item_.py

@@ -13,8 +13,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "167eb69066ad"
 down_revision = "1923519381ad"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2023-01-21-16.54.44_165d943c64ee_add_related_user_to_mealplan.py

@@ -14,8 +14,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "165d943c64ee"
 down_revision = "167eb69066ad"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2023-02-07-20.57.21_ff5f73b01a7a_add_missing_foreign_key_and_order_.py

@@ -11,8 +11,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "ff5f73b01a7a"
 down_revision = "165d943c64ee"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2023-02-10-21.18.32_16160bf731a0_add_more_indices_necessary_for_search.py

@@ -11,8 +11,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "16160bf731a0"
 down_revision = "ff5f73b01a7a"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2023-02-14-20.45.41_5ab195a474eb_add_normalized_search_properties.py

@@ -17,8 +17,8 @@ from mealie.db.models._model_utils.guid import GUID
 # revision identifiers, used by Alembic.
 revision = "5ab195a474eb"
 down_revision = "16160bf731a0"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 class SqlAlchemyBase(DeclarativeBase):

alembic/versions/2023-02-21-22.03.19_b04a08da2108_added_shopping_list_label_settings.py

@@ -18,8 +18,8 @@ from mealie.db.models._model_utils.guid import GUID
 # revision identifiers, used by Alembic.
 revision = "b04a08da2108"
 down_revision = "5ab195a474eb"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 # Intermediate table definitions

alembic/versions/2023-02-22-21.45.52_38514b39a824_add_auth_method_to_user_table.py

@@ -13,8 +13,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "38514b39a824"
 down_revision = "b04a08da2108"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def is_postgres():

alembic/versions/2023-04-13-06.47.04_b3dbb554ba53_postgres_fuzzy_search.py

@@ -11,8 +11,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "b3dbb554ba53"
 down_revision = "38514b39a824"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def get_db_type():

alembic/versions/2023-08-06-21.00.34_04ac51cbe9a4_added_group_slug.py

@@ -16,8 +16,8 @@ from mealie.db.models.group.group import Group
 # revision identifiers, used by Alembic.
 revision = "04ac51cbe9a4"
 down_revision = "b3dbb554ba53"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def populate_group_slugs(session: Session):

alembic/versions/2023-08-14-19.30.49_1825b5225403_added_recipe_note_to_shopping_list_.py

@@ -13,8 +13,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "1825b5225403"
 down_revision = "04ac51cbe9a4"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2023-08-15-16.25.07_bcfdad6b7355_remove_tool_name_and_slug_unique_.py

@@ -11,8 +11,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "bcfdad6b7355"
 down_revision = "1825b5225403"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2023-09-01-14.55.42_0341b154f79a_added_normalized_unit_and_food_names.py

@@ -15,8 +15,8 @@ from mealie.db.models.recipe.ingredient import IngredientFoodModel, IngredientUn
 # revision identifiers, used by Alembic.
 revision = "0341b154f79a"
 down_revision = "bcfdad6b7355"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def populate_normalized_fields():

alembic/versions/2023-10-04-14.29.26_dded3119c1fe_added_unique_constraints.py

@@ -23,8 +23,8 @@ from mealie.db.models.recipe.ingredient import IngredientFoodModel, IngredientUn
 # revision identifiers, used by Alembic.
 revision = "dded3119c1fe"
 down_revision = "0341b154f79a"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 # Intermediate table definitions

alembic/versions/2023-10-19-19.22.55_ba1e4a6cfe99_added_plural_names_and_alias_tables_for_.py

@@ -14,8 +14,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "ba1e4a6cfe99"
 down_revision = "dded3119c1fe"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2024-02-23-16.15.07_2298bb460ffd_added_user_to_shopping_list.py

@@ -20,8 +20,8 @@ logger = get_logger()
 # revision identifiers, used by Alembic.
 revision = "2298bb460ffd"
 down_revision = "ba1e4a6cfe99"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def is_postgres():

alembic/versions/2024-03-10-05.08.32_09aba125b57a_add_oidc_auth_method.py

@@ -11,8 +11,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "09aba125b57a"
 down_revision = "2298bb460ffd"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def is_postgres():

alembic/versions/2024-03-18-02.28.15_d7c6efd2de42_migrate_favorites_and_ratings_to_user_.py

@@ -20,8 +20,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "d7c6efd2de42"
 down_revision = "09aba125b57a"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def is_postgres():
@@ -202,8 +202,6 @@ def downgrade():
     )
     op.drop_index(op.f("ix_recipes_rating"), table_name="recipes")
     op.alter_column("recipes", "rating", existing_type=sa.Float(), type_=sa.INTEGER(), existing_nullable=True)
-    op.create_unique_constraint("ingredient_units_name_group_id_key", "ingredient_units", ["name", "group_id"])
-    op.create_unique_constraint("ingredient_foods_name_group_id_key", "ingredient_foods", ["name", "group_id"])
     op.create_table(
         "users_to_favorites",
         sa.Column("user_id", sa.CHAR(length=32), nullable=True),

alembic/versions/2024-04-07-01.05.20_7788478a0338_add_group_recipe_actions.py

@@ -14,8 +14,8 @@ from alembic import op
 # revision identifiers, used by Alembic.
 revision = "7788478a0338"
 down_revision = "d7c6efd2de42"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def upgrade():

alembic/versions/2024-06-22-10.17.03_32d69327997b_add_staple_flag_to_foods.py

@@ -11,12 +11,11 @@ from sqlalchemy import orm
 
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "32d69327997b"
 down_revision = "7788478a0338"
-branch_labels = None
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None
+depends_on: str | tuple[str, ...] | None = None
 
 
 def is_postgres():

alembic/versions/2024-07-12-16.16.29_feecc8ffb956_add_households.py

@@ -22,8 +22,8 @@ from mealie.core.config import get_app_settings
 # revision identifiers, used by Alembic.
 revision = "feecc8ffb956"
 down_revision = "32d69327997b"
-branch_labels = None  # type: ignore
+branch_labels: str | tuple[str, ...] | None = None
-depends_on = None  # type: ignore
+depends_on: str | tuple[str, ...] | None = None
 
 settings = get_app_settings()
 

docs/docs/contributors/developers-guide/database-changes.md

@@ -11,75 +11,5 @@ How exactly you need to modify it is of course highly contextual to the change y
 
 In your dev container you can run something like (change the message) `task py:migrate -- "Add creation tag to group preferences"` to have Alembic generate an upgrade script for you.
 
-The script Alembic generates, will be limited! (Perhaps there's a way to resolve that? Haven't looked into it yet)
+Alembic's script migration isn't perfect, so you will need to review which changes are generated. You will also need to make sure any custom operations work on both SQLite and Postgres.
-For example, Alembic generated a script _similar_ to this (it has been modified already to have accurate foreign key names, for instance):
+There are some known limitations with our migrations and Alembic's auto-generation, which is accounted for in `/alembic/env.py`. If any of your migrations overlap with the columns in `include_object`, you may need to manually adjust the migration.
-
-```Python
-"""Add creation tag to group preferences
-
-Revision ID: 0ea6eb8eaa44
-Revises: ba1e4a6cfe99
-Create Date: 2024-01-04 12:40:03.062671
-
-"""
-import sqlalchemy as sa
-
-import mealie.db.migration_types
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "0ea6eb8eaa44"
-down_revision = "ba1e4a6cfe99"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.add_column(
-        "group_preferences", sa.Column("recipe_creation_tag", mealie.db.migration_types.GUID(), nullable=True)
-    )
-    op.create_foreign_key("fk_groupprefs_tags", "group_preferences", "tags", ["recipe_creation_tag"], ["id"])
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_constraint("fk_groupprefs_tags", "group_preferences", type_="foreignkey")
-    op.drop_column("group_preferences", "recipe_creation_tag")
-    ### end Alembic commands ###
-```
-
-But when trying to actually use that upgrade script, it becomes clear that our SQLite database doesn't like them. The minor modification needed looks like:
-
-```Python
-"""Add creation tag to group preferences
-
-Revision ID: 0ea6eb8eaa44
-Revises: ba1e4a6cfe99
-Create Date: 2024-01-04 12:40:03.062671
-
-"""
-import sqlalchemy as sa
-
-import mealie.db.migration_types
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "0ea6eb8eaa44"
-down_revision = "ba1e4a6cfe99"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("group_preferences", schema=None) as batch_op:
-        batch_op.add_column(sa.Column("recipe_creation_tag", mealie.db.migration_types.GUID(), nullable=True))
-        batch_op.create_foreign_key("fk_groupprefs_tags", "tags", ["recipe_creation_tag"], ["id"])
-
-
-def downgrade():
-    with op.batch_alter_table("group_preferences", schema=None) as batch_op:
-        batch_op.drop_constraint("fk_groupprefs_tags", type_="foreignkey")
-        batch_op.drop_column("recipe_creation_tag")
-```