From 310069a7e9b8c87053deb1dac762c7ccc2ac2441 Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:37:19 -0600 Subject: [PATCH] fix: various alembic migration issues with queries (#2773) * set expire_on_commit false to avoid refresh * converted deletes to raw SQL statements * call update statements directly in sql * parameterized text queries * replace orm with raw sql to avoid db differences --- ...9a_added_normalized_unit_and_food_names.py | 22 ++++-- ...6_dded3119c1fe_added_unique_constraints.py | 74 +++++++------------ 2 files changed, 41 insertions(+), 55 deletions(-) diff --git a/alembic/versions/2023-09-01-14.55.42_0341b154f79a_added_normalized_unit_and_food_names.py b/alembic/versions/2023-09-01-14.55.42_0341b154f79a_added_normalized_unit_and_food_names.py index 44efe74faf84..f77fe6fbc21e 100644 --- a/alembic/versions/2023-09-01-14.55.42_0341b154f79a_added_normalized_unit_and_food_names.py +++ b/alembic/versions/2023-09-01-14.55.42_0341b154f79a_added_normalized_unit_and_food_names.py @@ -33,21 +33,29 @@ def populate_normalized_fields(): ) for unit in units: if unit.name is not None: - unit.name_normalized = IngredientUnitModel.normalize(unit.name) + session.execute( + sa.text( + f"UPDATE {IngredientUnitModel.__tablename__} SET name_normalized=:name_normalized WHERE id=:id" + ).bindparams(name_normalized=IngredientUnitModel.normalize(unit.name), id=unit.id) + ) if unit.abbreviation is not None: - unit.abbreviation_normalized = IngredientUnitModel.normalize(unit.abbreviation) - - session.add(unit) + session.execute( + sa.text( + f"UPDATE {IngredientUnitModel.__tablename__} SET abbreviation_normalized=:abbreviation_normalized WHERE id=:id" + ).bindparams(abbreviation_normalized=IngredientUnitModel.normalize(unit.abbreviation), id=unit.id) + ) foods = ( session.execute(select(IngredientFoodModel).options(orm.load_only(IngredientFoodModel.name))).scalars().all() ) for food in foods: if food.name is not None: - food.name_normalized = IngredientFoodModel.normalize(food.name) - - session.add(food) + session.execute( + sa.text( + f"UPDATE {IngredientFoodModel.__tablename__} SET name_normalized=:name_normalized WHERE id=:id" + ).bindparams(name_normalized=IngredientFoodModel.normalize(food.name), id=food.id) + ) session.commit() diff --git a/alembic/versions/2023-10-04-14.29.26_dded3119c1fe_added_unique_constraints.py b/alembic/versions/2023-10-04-14.29.26_dded3119c1fe_added_unique_constraints.py index 850404713a2d..885e619f8199 100644 --- a/alembic/versions/2023-10-04-14.29.26_dded3119c1fe_added_unique_constraints.py +++ b/alembic/versions/2023-10-04-14.29.26_dded3119c1fe_added_unique_constraints.py @@ -13,10 +13,8 @@ import sqlalchemy as sa from pydantic import UUID4 from sqlalchemy.orm import Session, load_only -import mealie.db.migration_types from alembic import op from mealie.db.models._model_base import SqlAlchemyBase -from mealie.db.models._model_utils.guid import GUID from mealie.db.models.group.shopping_list import ShoppingListItem from mealie.db.models.labels import MultiPurposeLabel from mealie.db.models.recipe.ingredient import IngredientFoodModel, IngredientUnitModel, RecipeIngredientModel @@ -43,26 +41,25 @@ def _is_postgres(): return op.get_context().dialect.name == "postgresql" -def _get_duplicates(session: Session, model: SqlAlchemyBase) -> defaultdict[str, list[str]]: - duplicate_map: defaultdict[str, list[str]] = defaultdict(list) - for obj in session.query(model).options(load_only(model.id, model.group_id, model.name)).all(): - key = f"{obj.group_id}$${obj.name}" - duplicate_map[key].append(str(obj.id)) +def _get_duplicates(session: Session, model: SqlAlchemyBase) -> defaultdict[str, list]: + duplicate_map: defaultdict[str, list] = defaultdict(list) + + query = session.execute(sa.text(f"SELECT id, group_id, name FROM {model.__tablename__}")) + for row in query.all(): + id, group_id, name = row + key = f"{group_id}$${name}" + duplicate_map[key].append(id) return duplicate_map def _resolve_duplicate_food( session: Session, - keep_food: IngredientFoodModel, keep_food_id: UUID4, dupe_food_id: UUID4, ): for shopping_list_item in session.query(ShoppingListItem).filter_by(food_id=dupe_food_id).all(): shopping_list_item.food_id = keep_food_id - shopping_list_item.food = keep_food - - session.commit() for recipe_ingredient in ( session.query(RecipeIngredientModel) @@ -71,62 +68,43 @@ def _resolve_duplicate_food( .all() ): recipe_ingredient.food_id = keep_food_id - recipe_ingredient.food = keep_food - session.commit() - - session.query(IngredientFoodModel).options(load_only(IngredientFoodModel.id)).filter_by(id=dupe_food_id).delete() - session.commit() + session.execute( + sa.text(f"DELETE FROM {IngredientFoodModel.__tablename__} WHERE id=:id").bindparams(id=dupe_food_id) + ) def _resolve_duplicate_unit( session: Session, - keep_unit: IngredientUnitModel, keep_unit_id: UUID4, dupe_unit_id: UUID4, ): for shopping_list_item in session.query(ShoppingListItem).filter_by(unit_id=dupe_unit_id).all(): shopping_list_item.unit_id = keep_unit_id - shopping_list_item.unit = keep_unit - - session.commit() for recipe_ingredient in session.query(RecipeIngredientModel).filter_by(unit_id=dupe_unit_id).all(): recipe_ingredient.unit_id = keep_unit_id - recipe_ingredient.unit = keep_unit - session.commit() - - session.query(IngredientUnitModel).options(load_only(IngredientUnitModel.id)).filter_by(id=dupe_unit_id).delete() - session.commit() + session.execute( + sa.text(f"DELETE FROM {IngredientUnitModel.__tablename__} WHERE id=:id").bindparams(id=dupe_unit_id) + ) def _resolve_duplicate_label( session: Session, - keep_label: MultiPurposeLabel, keep_label_id: UUID4, dupe_label_id: UUID4, ): for shopping_list_item in session.query(ShoppingListItem).filter_by(label_id=dupe_label_id).all(): shopping_list_item.label_id = keep_label_id - shopping_list_item.label = keep_label - - session.commit() for ingredient_food in session.query(IngredientFoodModel).filter_by(label_id=dupe_label_id).all(): ingredient_food.label_id = keep_label_id - ingredient_food.label = keep_label - session.commit() - - session.query(MultiPurposeLabel).options(load_only(MultiPurposeLabel.id)).filter_by(id=dupe_label_id).delete() - session.commit() + session.execute(sa.text(f"DELETE FROM {MultiPurposeLabel.__tablename__} WHERE id=:id").bindparams(id=dupe_label_id)) -def _resolve_duplicate_foods_units_labels(): - bind = op.get_bind() - session = Session(bind=bind) - +def _resolve_duplicate_foods_units_labels(session: Session): for model, resolve_func in [ (IngredientFoodModel, _resolve_duplicate_food), (IngredientUnitModel, _resolve_duplicate_unit), @@ -138,9 +116,8 @@ def _resolve_duplicate_foods_units_labels(): continue keep_id = ids[0] - keep_obj = session.query(model).options(load_only(model.id)).filter_by(id=keep_id).first() for dupe_id in ids[1:]: - resolve_func(session, keep_obj, keep_id, dupe_id) + resolve_func(session, keep_id, dupe_id) def _remove_duplicates_from_m2m_table(session: Session, table_meta: TableMeta): @@ -163,20 +140,20 @@ def _remove_duplicates_from_m2m_table(session: Session, table_meta: TableMeta): ) session.execute(query) - session.commit() -def _remove_duplicates_from_m2m_tables(table_metas: list[TableMeta]): - bind = op.get_bind() - session = Session(bind=bind) - +def _remove_duplicates_from_m2m_tables(session: Session, table_metas: list[TableMeta]): for table_meta in table_metas: _remove_duplicates_from_m2m_table(session, table_meta) def upgrade(): - _resolve_duplicate_foods_units_labels() + bind = op.get_bind() + session = Session(bind=bind) + + _resolve_duplicate_foods_units_labels(session) _remove_duplicates_from_m2m_tables( + session, [ TableMeta("cookbooks_to_categories", "cookbook_id", "category_id"), TableMeta("cookbooks_to_tags", "cookbook_id", "tag_id"), @@ -189,12 +166,13 @@ def upgrade(): TableMeta("recipes_to_tools", "recipe_id", "tool_id"), TableMeta("users_to_favorites", "user_id", "recipe_id"), TableMeta("shopping_lists_multi_purpose_labels", "shopping_list_id", "label_id"), - ] + ], ) + session.commit() + # ### commands auto generated by Alembic - please adjust! ### # we use batch_alter_table here because otherwise this fails on sqlite - # M2M with op.batch_alter_table("cookbooks_to_categories") as batch_op: batch_op.create_unique_constraint("cookbook_id_category_id_key", ["cookbook_id", "category_id"])