Remove all sqlalchemy lazy-loading from app (#2260)

* Remove some implicit lazy-loads from user serialization

* implement full backup restore across different database versions

* rework all custom getter dicts to not leak lazy loads

* remove some occurances of lazy-loading

* remove a lot of lazy loading from recipes

* add more eager loading
remove loading options from repository
remove raiseload for checking

* fix failing test

* do not apply loader options for paging counts

* try using selectinload a bit more instead of joinedload

* linter fixes

mealie/db/models/recipe/ingredient.py

@@ -28,7 +28,9 @@ class IngredientUnitModel(SqlAlchemyBase, BaseMixins):
     abbreviation: Mapped[str | None] = mapped_column(String)
     use_abbreviation: Mapped[bool | None] = mapped_column(Boolean, default=False)
     fraction: Mapped[bool | None] = mapped_column(Boolean, default=True)
-    ingredients: Mapped[list["RecipeIngredient"]] = orm.relationship("RecipeIngredient", back_populates="unit")
+    ingredients: Mapped[list["RecipeIngredientModel"]] = orm.relationship(
+        "RecipeIngredientModel", back_populates="unit"
+    )
 
     @auto_init()
     def __init__(self, **_) -> None:
@@ -45,7 +47,9 @@ class IngredientFoodModel(SqlAlchemyBase, BaseMixins):
 
     name: Mapped[str | None] = mapped_column(String)
     description: Mapped[str | None] = mapped_column(String)
-    ingredients: Mapped[list["RecipeIngredient"]] = orm.relationship("RecipeIngredient", back_populates="food")
+    ingredients: Mapped[list["RecipeIngredientModel"]] = orm.relationship(
+        "RecipeIngredientModel", back_populates="food"
+    )
     extras: Mapped[list[IngredientFoodExtras]] = orm.relationship("IngredientFoodExtras", cascade="all, delete-orphan")
 
     label_id: Mapped[GUID | None] = mapped_column(GUID, ForeignKey("multi_purpose_labels.id"), index=True)
@@ -57,7 +61,7 @@ class IngredientFoodModel(SqlAlchemyBase, BaseMixins):
         pass
 
 
-class RecipeIngredient(SqlAlchemyBase, BaseMixins):
+class RecipeIngredientModel(SqlAlchemyBase, BaseMixins):
     __tablename__ = "recipes_ingredients"
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
     position: Mapped[int | None] = mapped_column(Integer, index=True)
@@ -92,16 +96,16 @@ class RecipeIngredient(SqlAlchemyBase, BaseMixins):
             self.orginal_text = unidecode(orginal_text).lower().strip()
 
 
-@event.listens_for(RecipeIngredient.note, "set")
+@event.listens_for(RecipeIngredientModel.note, "set")
-def receive_note(target: RecipeIngredient, value: str, oldvalue, initiator):
+def receive_note(target: RecipeIngredientModel, value: str, oldvalue, initiator):
     if value is not None:
         target.name_normalized = unidecode(value).lower().strip()
     else:
         target.name_normalized = None
 
 
-@event.listens_for(RecipeIngredient.original_text, "set")
+@event.listens_for(RecipeIngredientModel.original_text, "set")
-def receive_original_text(target: RecipeIngredient, value: str, oldvalue, initiator):
+def receive_original_text(target: RecipeIngredientModel, value: str, oldvalue, initiator):
     if value is not None:
         target.original_text_normalized = unidecode(value).lower().strip()
     else:

mealie/db/models/recipe/recipe.py

@@ -17,7 +17,7 @@ from .api_extras import ApiExtras, api_extras
 from .assets import RecipeAsset
 from .category import recipes_to_categories
 from .comment import RecipeComment
-from .ingredient import RecipeIngredient
+from .ingredient import RecipeIngredientModel
 from .instruction import RecipeInstruction
 from .note import Note
 from .nutrition import Nutrition
@@ -77,10 +77,10 @@ class RecipeModel(SqlAlchemyBase, BaseMixins):
     )
     tools: Mapped[list["Tool"]] = orm.relationship("Tool", secondary=recipes_to_tools, back_populates="recipes")
 
-    recipe_ingredient: Mapped[list[RecipeIngredient]] = orm.relationship(
+    recipe_ingredient: Mapped[list[RecipeIngredientModel]] = orm.relationship(
-        "RecipeIngredient",
+        "RecipeIngredientModel",
         cascade="all, delete-orphan",
-        order_by="RecipeIngredient.position",
+        order_by="RecipeIngredientModel.position",
         collection_class=ordering_list("position"),
     )
     recipe_instructions: Mapped[list[RecipeInstruction]] = orm.relationship(
@@ -173,7 +173,7 @@ class RecipeModel(SqlAlchemyBase, BaseMixins):
             self.recipe_instructions = [RecipeInstruction(**step, session=session) for step in recipe_instructions]
 
         if recipe_ingredient is not None:
-            self.recipe_ingredient = [RecipeIngredient(**ingr, session=session) for ingr in recipe_ingredient]
+            self.recipe_ingredient = [RecipeIngredientModel(**ingr, session=session) for ingr in recipe_ingredient]
 
         if assets:
             self.assets = [RecipeAsset(**a) for a in assets]

mealie/repos/repository_foods.py

@@ -1,7 +1,5 @@
 from pydantic import UUID4
 from sqlalchemy import select
-from sqlalchemy.orm import joinedload
-from sqlalchemy.orm.interfaces import LoaderOption
 
 from mealie.db.models.recipe.ingredient import IngredientFoodModel
 from mealie.schema.recipe.recipe_ingredient import IngredientFood
@@ -31,9 +29,3 @@ class RepositoryFood(RepositoryGeneric[IngredientFood, IngredientFoodModel]):
 
     def by_group(self, group_id: UUID4) -> "RepositoryFood":
         return super().by_group(group_id)
-
-    def paging_query_options(self) -> list[LoaderOption]:
-        return [
-            joinedload(IngredientFoodModel.extras),
-            joinedload(IngredientFoodModel.label),
-        ]

mealie/repos/repository_generic.py

@@ -7,16 +7,16 @@ from typing import Any, Generic, TypeVar
 from fastapi import HTTPException
 from pydantic import UUID4, BaseModel
 from sqlalchemy import Select, delete, func, select
-from sqlalchemy.orm.interfaces import LoaderOption
 from sqlalchemy.orm.session import Session
 from sqlalchemy.sql import sqltypes
 
 from mealie.core.root_logger import get_logger
 from mealie.db.models._model_base import SqlAlchemyBase
+from mealie.schema._mealie import MealieModel
 from mealie.schema.response.pagination import OrderDirection, PaginationBase, PaginationQuery
 from mealie.schema.response.query_filter import QueryFilter
 
-Schema = TypeVar("Schema", bound=BaseModel)
+Schema = TypeVar("Schema", bound=MealieModel)
 Model = TypeVar("Model", bound=SqlAlchemyBase)
 
 T = TypeVar("T", bound="RepositoryGeneric")
@@ -54,8 +54,13 @@ class RepositoryGeneric(Generic[Schema, Model]):
         self.logger.error(f"Error processing query for Repo model={self.model.__name__} schema={self.schema.__name__}")
         self.logger.error(e)
 
-    def _query(self):
+    def _query(self, override_schema: type[MealieModel] | None = None, with_options=True):
-        return select(self.model)
+        q = select(self.model)
+        if with_options:
+            schema = override_schema or self.schema
+            return q.options(*schema.loader_options())
+        else:
+            return q
 
     def _filter_builder(self, **kwargs) -> dict[str, Any]:
         dct = {}
@@ -83,7 +88,7 @@ class RepositoryGeneric(Generic[Schema, Model]):
 
         fltr = self._filter_builder()
 
-        q = self._query().filter_by(**fltr)
+        q = self._query(override_schema=eff_schema).filter_by(**fltr)
 
         if order_by:
             try:
@@ -98,7 +103,7 @@ class RepositoryGeneric(Generic[Schema, Model]):
 
             except AttributeError:
                 self.logger.info(f'Attempted to sort by unknown sort property "{order_by}"; ignoring')
-        result = self.session.execute(q.offset(start).limit(limit)).scalars().all()
+        result = self.session.execute(q.offset(start).limit(limit)).unique().scalars().all()
         return [eff_schema.from_orm(x) for x in result]
 
     def multi_query(
@@ -113,7 +118,7 @@ class RepositoryGeneric(Generic[Schema, Model]):
         eff_schema = override_schema or self.schema
 
         fltr = self._filter_builder(**query_by)
-        q = self._query().filter_by(**fltr)
+        q = self._query(override_schema=eff_schema).filter_by(**fltr)
 
         if order_by:
             if order_attr := getattr(self.model, str(order_by)):
@@ -121,7 +126,7 @@ class RepositoryGeneric(Generic[Schema, Model]):
                 q = q.order_by(order_attr)
 
         q = q.offset(start).limit(limit)
-        result = self.session.execute(q).scalars().all()
+        result = self.session.execute(q).unique().scalars().all()
         return [eff_schema.from_orm(x) for x in result]
 
     def _query_one(self, match_value: str | int | UUID4, match_key: str | None = None) -> Model:
@@ -133,14 +138,15 @@ class RepositoryGeneric(Generic[Schema, Model]):
             match_key = self.primary_key
 
         fltr = self._filter_builder(**{match_key: match_value})
-        return self.session.execute(self._query().filter_by(**fltr)).scalars().one()
+        return self.session.execute(self._query().filter_by(**fltr)).unique().scalars().one()
 
     def get_one(
         self, value: str | int | UUID4, key: str | None = None, any_case=False, override_schema=None
     ) -> Schema | None:
         key = key or self.primary_key
+        eff_schema = override_schema or self.schema
 
-        q = self._query()
+        q = self._query(override_schema=eff_schema)
 
         if any_case:
             search_attr = getattr(self.model, key)
@@ -148,12 +154,11 @@ class RepositoryGeneric(Generic[Schema, Model]):
         else:
             q = q.filter_by(**self._filter_builder(**{key: value}))
 
-        result = self.session.execute(q).scalars().one_or_none()
+        result = self.session.execute(q).unique().scalars().one_or_none()
 
         if not result:
             return None
 
-        eff_schema = override_schema or self.schema
         return eff_schema.from_orm(result)
 
     def create(self, data: Schema | BaseModel | dict) -> Schema:
@@ -205,7 +210,7 @@ class RepositoryGeneric(Generic[Schema, Model]):
             document_data_by_id[document_data["id"]] = document_data
 
         documents_to_update_query = self._query().filter(self.model.id.in_(list(document_data_by_id.keys())))
-        documents_to_update = self.session.execute(documents_to_update_query).scalars().all()
+        documents_to_update = self.session.execute(documents_to_update_query).unique().scalars().all()
 
         updated_documents = []
         for document_to_update in documents_to_update:
@@ -229,7 +234,7 @@ class RepositoryGeneric(Generic[Schema, Model]):
     def delete(self, value, match_key: str | None = None) -> Schema:
         match_key = match_key or self.primary_key
 
-        result = self.session.execute(self._query().filter_by(**{match_key: value})).scalars().one()
+        result = self._query_one(value, match_key)
         results_as_model = self.schema.from_orm(result)
 
         try:
@@ -243,7 +248,7 @@ class RepositoryGeneric(Generic[Schema, Model]):
 
     def delete_many(self, values: Iterable) -> Schema:
         query = self._query().filter(self.model.id.in_(values))  # type: ignore
-        results = self.session.execute(query).scalars().all()
+        results = self.session.execute(query).unique().scalars().all()
         results_as_model = [self.schema.from_orm(result) for result in results]
 
         try:
@@ -282,13 +287,9 @@ class RepositoryGeneric(Generic[Schema, Model]):
             q = select(func.count(self.model.id)).filter(attribute_name == attr_match)
             return self.session.scalar(q)
         else:
-            q = self._query().filter(attribute_name == attr_match)
+            q = self._query(override_schema=eff_schema).filter(attribute_name == attr_match)
             return [eff_schema.from_orm(x) for x in self.session.execute(q).scalars().all()]
 
-    def paging_query_options(self) -> list[LoaderOption]:
-        # Override this in subclasses to specify joinedloads or similar for page_all
-        return []
-
     def page_all(self, pagination: PaginationQuery, override=None) -> PaginationBase[Schema]:
         """
         pagination is a method to interact with the filtered database table and return a paginated result
@@ -301,12 +302,14 @@ class RepositoryGeneric(Generic[Schema, Model]):
         """
         eff_schema = override or self.schema
 
-        q = self._query().options(*self.paging_query_options())
+        q = self._query(override_schema=eff_schema, with_options=False)
 
         fltr = self._filter_builder()
         q = q.filter_by(**fltr)
         q, count, total_pages = self.add_pagination_to_query(q, pagination)
 
+        # Apply options late, so they do not get used for counting
+        q = q.options(*eff_schema.loader_options())
         try:
             data = self.session.execute(q).unique().scalars().all()
         except Exception as e:

mealie/repos/repository_recipes.py

@@ -10,7 +10,7 @@ from sqlalchemy.orm import joinedload
 from text_unidecode import unidecode
 
 from mealie.db.models.recipe.category import Category
-from mealie.db.models.recipe.ingredient import RecipeIngredient
+from mealie.db.models.recipe.ingredient import RecipeIngredientModel
 from mealie.db.models.recipe.recipe import RecipeModel
 from mealie.db.models.recipe.settings import RecipeSettings
 from mealie.db.models.recipe.tag import Tag
@@ -108,7 +108,7 @@ class RepositoryRecipes(RepositoryGeneric[Recipe, RecipeModel]):
         ]
 
         if load_foods:
-            args.append(joinedload(RecipeModel.recipe_ingredient).options(joinedload(RecipeIngredient.food)))
+            args.append(joinedload(RecipeModel.recipe_ingredient).options(joinedload(RecipeIngredientModel.food)))
 
         try:
             if order_by:
@@ -156,10 +156,10 @@ class RepositoryRecipes(RepositoryGeneric[Recipe, RecipeModel]):
         # that at least sqlite wont use indexes for that correctly anymore and takes a big hit, so prefiltering it is
         ingredient_ids = (
             self.session.execute(
-                select(RecipeIngredient.id).filter(
+                select(RecipeIngredientModel.id).filter(
                     or_(
-                        RecipeIngredient.note_normalized.like(f"%{normalized_search}%"),
+                        RecipeIngredientModel.note_normalized.like(f"%{normalized_search}%"),
-                        RecipeIngredient.original_text_normalized.like(f"%{normalized_search}%"),
+                        RecipeIngredientModel.original_text_normalized.like(f"%{normalized_search}%"),
                     )
                 )
             )
@@ -171,7 +171,7 @@ class RepositoryRecipes(RepositoryGeneric[Recipe, RecipeModel]):
             or_(
                 RecipeModel.name_normalized.like(f"%{normalized_search}%"),
                 RecipeModel.description_normalized.like(f"%{normalized_search}%"),
-                RecipeModel.recipe_ingredient.any(RecipeIngredient.id.in_(ingredient_ids)),
+                RecipeModel.recipe_ingredient.any(RecipeIngredientModel.id.in_(ingredient_ids)),
             )
         ).order_by(desc(RecipeModel.name_normalized.like(f"%{normalized_search}%")))
         return q
@@ -303,9 +303,9 @@ class RepositoryRecipes(RepositoryGeneric[Recipe, RecipeModel]):
                 fltr.append(RecipeModel.tools.any(Tool.id.in_(tools)))
         if foods:
             if require_all_foods:
-                fltr.extend(RecipeModel.recipe_ingredient.any(RecipeIngredient.food_id == food) for food in foods)
+                fltr.extend(RecipeModel.recipe_ingredient.any(RecipeIngredientModel.food_id == food) for food in foods)
             else:
-                fltr.append(RecipeModel.recipe_ingredient.any(RecipeIngredient.food_id.in_(foods)))
+                fltr.append(RecipeModel.recipe_ingredient.any(RecipeIngredientModel.food_id.in_(foods)))
         return fltr
 
     def by_category_and_tags(

mealie/repos/repository_users.py

@@ -5,8 +5,9 @@ from pydantic import UUID4
 from sqlalchemy import select
 
 from mealie.assets import users as users_assets
-from mealie.schema.user.user import PrivateUser, User
+from mealie.schema.user.user import PrivateUser
 
+from ..db.models.users import User
 from .repository_generic import RepositoryGeneric
 
 

mealie/schema/_mealie/mealie_model.py

@@ -5,6 +5,7 @@ from typing import Protocol, TypeVar
 
 from humps.main import camelize
 from pydantic import UUID4, BaseModel
+from sqlalchemy.orm.interfaces import LoaderOption
 
 T = TypeVar("T", bound=BaseModel)
 
@@ -54,6 +55,10 @@ class MealieModel(BaseModel):
             if field in self.__fields__ and (val is not None or replace_null):
                 setattr(self, field, val)
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return []
+
 
 class HasUUID(Protocol):
     id: UUID4

mealie/schema/cookbook/cookbook.py

@@ -1,10 +1,13 @@
 from pydantic import UUID4, validator
 from slugify import slugify
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm.interfaces import LoaderOption
 
 from mealie.schema._mealie import MealieModel
 from mealie.schema.recipe.recipe import RecipeSummary, RecipeTool
 from mealie.schema.response.pagination import PaginationBase
 
+from ...db.models.group import CookBook
 from ..recipe.recipe_category import CategoryBase, TagBase
 
 
@@ -51,6 +54,10 @@ class ReadCookBook(UpdateCookBook):
     class Config:
         orm_mode = True
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [joinedload(CookBook.categories), joinedload(CookBook.tags), joinedload(CookBook.tools)]
+
 
 class CookBookPagination(PaginationBase):
     items: list[ReadCookBook]

mealie/schema/getter_dict.py

@@ -0,0 +1,33 @@
+from collections.abc import Callable, Mapping
+from typing import Any
+
+from pydantic.utils import GetterDict
+
+
+class CustomGetterDict(GetterDict):
+    transformations: Mapping[str, Callable[[Any], Any]]
+
+    def get(self, key: Any, default: Any = None) -> Any:
+        # Transform extras into key-value dict
+        if key in self.transformations:
+            value = super().get(key, default)
+            return self.transformations[key](value)
+
+        # Keep all other fields as they are
+        else:
+            return super().get(key, default)
+
+
+class ExtrasGetterDict(CustomGetterDict):
+    transformations = {"extras": lambda value: {x.key_name: x.value for x in value}}
+
+
+class GroupGetterDict(CustomGetterDict):
+    transformations = {"group": lambda value: value.name}
+
+
+class UserGetterDict(CustomGetterDict):
+    transformations = {
+        "group": lambda value: value.name,
+        "favorite_recipes": lambda value: [x.slug for x in value],
+    }

mealie/schema/group/group_events.py

@@ -1,5 +1,8 @@
 from pydantic import UUID4, NoneStr
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm.interfaces import LoaderOption
 
+from mealie.db.models.group import GroupEventNotifierModel
 from mealie.schema._mealie import MealieModel
 from mealie.schema.response.pagination import PaginationBase
 
@@ -86,6 +89,10 @@ class GroupEventNotifierOut(MealieModel):
     class Config:
         orm_mode = True
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [joinedload(GroupEventNotifierModel.options)]
+
 
 class GroupEventPagination(PaginationBase):
     items: list[GroupEventNotifierOut]

mealie/schema/group/group_shopping_list.py

@@ -4,11 +4,19 @@ from datetime import datetime
 from fractions import Fraction
 
 from pydantic import UUID4, validator
-from pydantic.utils import GetterDict
+from sqlalchemy.orm import joinedload, selectinload
+from sqlalchemy.orm.interfaces import LoaderOption
 
-from mealie.db.models.group.shopping_list import ShoppingList, ShoppingListItem
+from mealie.db.models.group import (
+    ShoppingList,
+    ShoppingListItem,
+    ShoppingListMultiPurposeLabel,
+    ShoppingListRecipeReference,
+)
+from mealie.db.models.recipe import IngredientFoodModel, RecipeModel
 from mealie.schema._mealie import MealieModel
 from mealie.schema._mealie.types import NoneFloat
+from mealie.schema.getter_dict import ExtrasGetterDict
 from mealie.schema.labels.multi_purpose_label import MultiPurposeLabelSummary
 from mealie.schema.recipe.recipe import RecipeSummary
 from mealie.schema.recipe.recipe_ingredient import (
@@ -171,13 +179,18 @@ class ShoppingListItemOut(ShoppingListItemBase):
 
     class Config:
         orm_mode = True
+        getter_dict = ExtrasGetterDict
 
-        @classmethod
+    @classmethod
-        def getter_dict(cls, name_orm: ShoppingListItem):
+    def loader_options(cls) -> list[LoaderOption]:
-            return {
+        return [
-                **GetterDict(name_orm),
+            selectinload(ShoppingListItem.extras),
-                "extras": {x.key_name: x.value for x in name_orm.extras},
+            selectinload(ShoppingListItem.food).joinedload(IngredientFoodModel.extras),
-            }
+            selectinload(ShoppingListItem.food).joinedload(IngredientFoodModel.label),
+            joinedload(ShoppingListItem.label),
+            joinedload(ShoppingListItem.unit),
+            selectinload(ShoppingListItem.recipe_references),
+        ]
 
 
 class ShoppingListItemsCollectionOut(MealieModel):
@@ -204,6 +217,10 @@ class ShoppingListMultiPurposeLabelOut(ShoppingListMultiPurposeLabelUpdate):
     class Config:
         orm_mode = True
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [joinedload(ShoppingListMultiPurposeLabel.label)]
+
 
 class ShoppingListItemPagination(PaginationBase):
     items: list[ShoppingListItemOut]
@@ -229,6 +246,14 @@ class ShoppingListRecipeRefOut(MealieModel):
     class Config:
         orm_mode = True
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [
+            selectinload(ShoppingListRecipeReference.recipe).joinedload(RecipeModel.recipe_category),
+            selectinload(ShoppingListRecipeReference.recipe).joinedload(RecipeModel.tags),
+            selectinload(ShoppingListRecipeReference.recipe).joinedload(RecipeModel.tools),
+        ]
+
 
 class ShoppingListSave(ShoppingListCreate):
     group_id: UUID4
@@ -241,13 +266,23 @@ class ShoppingListSummary(ShoppingListSave):
 
     class Config:
         orm_mode = True
+        getter_dict = ExtrasGetterDict
 
-        @classmethod
+    @classmethod
-        def getter_dict(cls, name_orm: ShoppingList):
+    def loader_options(cls) -> list[LoaderOption]:
-            return {
+        return [
-                **GetterDict(name_orm),
+            selectinload(ShoppingList.extras),
-                "extras": {x.key_name: x.value for x in name_orm.extras},
+            selectinload(ShoppingList.recipe_references)
-            }
+            .joinedload(ShoppingListRecipeReference.recipe)
+            .joinedload(RecipeModel.recipe_category),
+            selectinload(ShoppingList.recipe_references)
+            .joinedload(ShoppingListRecipeReference.recipe)
+            .joinedload(RecipeModel.tags),
+            selectinload(ShoppingList.recipe_references)
+            .joinedload(ShoppingListRecipeReference.recipe)
+            .joinedload(RecipeModel.tools),
+            selectinload(ShoppingList.label_settings).joinedload(ShoppingListMultiPurposeLabel.label),
+        ]
 
 
 class ShoppingListPagination(PaginationBase):
@@ -265,13 +300,33 @@ class ShoppingListOut(ShoppingListUpdate):
 
     class Config:
         orm_mode = True
+        getter_dict = ExtrasGetterDict
 
-        @classmethod
+    @classmethod
-        def getter_dict(cls, name_orm: ShoppingList):
+    def loader_options(cls) -> list[LoaderOption]:
-            return {
+        return [
-                **GetterDict(name_orm),
+            selectinload(ShoppingList.extras),
-                "extras": {x.key_name: x.value for x in name_orm.extras},
+            selectinload(ShoppingList.list_items).joinedload(ShoppingListItem.extras),
-            }
+            selectinload(ShoppingList.list_items)
+            .joinedload(ShoppingListItem.food)
+            .joinedload(IngredientFoodModel.extras),
+            selectinload(ShoppingList.list_items)
+            .joinedload(ShoppingListItem.food)
+            .joinedload(IngredientFoodModel.label),
+            selectinload(ShoppingList.list_items).joinedload(ShoppingListItem.label),
+            selectinload(ShoppingList.list_items).joinedload(ShoppingListItem.unit),
+            selectinload(ShoppingList.list_items).joinedload(ShoppingListItem.recipe_references),
+            selectinload(ShoppingList.recipe_references)
+            .joinedload(ShoppingListRecipeReference.recipe)
+            .joinedload(RecipeModel.recipe_category),
+            selectinload(ShoppingList.recipe_references)
+            .joinedload(ShoppingListRecipeReference.recipe)
+            .joinedload(RecipeModel.tags),
+            selectinload(ShoppingList.recipe_references)
+            .joinedload(ShoppingListRecipeReference.recipe)
+            .joinedload(RecipeModel.tools),
+            selectinload(ShoppingList.label_settings).joinedload(ShoppingListMultiPurposeLabel.label),
+        ]
 
 
 class ShoppingListAddRecipeParams(MealieModel):

mealie/schema/meal_plan/new_meal.py

@@ -3,7 +3,11 @@ from enum import Enum
 from uuid import UUID
 
 from pydantic import validator
+from sqlalchemy.orm import selectinload
+from sqlalchemy.orm.interfaces import LoaderOption
 
+from mealie.db.models.group import GroupMealPlan
+from mealie.db.models.recipe import RecipeModel
 from mealie.schema._mealie import MealieModel
 from mealie.schema.recipe.recipe import RecipeSummary
 from mealie.schema.response.pagination import PaginationBase
@@ -57,6 +61,14 @@ class ReadPlanEntry(UpdatePlanEntry):
     class Config:
         orm_mode = True
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [
+            selectinload(GroupMealPlan.recipe).joinedload(RecipeModel.recipe_category),
+            selectinload(GroupMealPlan.recipe).joinedload(RecipeModel.tags),
+            selectinload(GroupMealPlan.recipe).joinedload(RecipeModel.tools),
+        ]
+
 
 class PlanEntryPagination(PaginationBase):
     items: list[ReadPlanEntry]

mealie/schema/meal_plan/plan_rules.py

@@ -2,7 +2,10 @@ import datetime
 from enum import Enum
 
 from pydantic import UUID4
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm.interfaces import LoaderOption
 
+from mealie.db.models.group import GroupMealPlanRules
 from mealie.schema._mealie import MealieModel
 from mealie.schema.response.pagination import PaginationBase
 
@@ -65,6 +68,10 @@ class PlanRulesOut(PlanRulesSave):
     class Config:
         orm_mode = True
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [joinedload(GroupMealPlanRules.categories), joinedload(GroupMealPlanRules.tags)]
+
 
 class PlanRulesPagination(PaginationBase):
     items: list[PlanRulesOut]

mealie/schema/meal_plan/shopping_list.py

@@ -1,7 +1,5 @@
-from pydantic.utils import GetterDict
-
-from mealie.db.models.group.shopping_list import ShoppingList
 from mealie.schema._mealie import MealieModel
+from mealie.schema.getter_dict import GroupGetterDict
 
 
 class ListItem(MealieModel):
@@ -25,10 +23,4 @@ class ShoppingListOut(ShoppingListIn):
 
     class Config:
         orm_mode = True
-
+        getter_dict = GroupGetterDict
-        @classmethod
-        def getter_dict(cls, ormModel: ShoppingList):
-            return {
-                **GetterDict(ormModel),
-                "group": ormModel.group.name,
-            }

mealie/schema/recipe/recipe.py

@@ -6,14 +6,22 @@ from typing import Any
 from uuid import uuid4
 
 from pydantic import UUID4, BaseModel, Field, validator
-from pydantic.utils import GetterDict
 from slugify import slugify
+from sqlalchemy.orm import joinedload, selectinload
+from sqlalchemy.orm.interfaces import LoaderOption
 
 from mealie.core.config import get_app_dirs
-from mealie.db.models.recipe.recipe import RecipeModel
 from mealie.schema._mealie import MealieModel
 from mealie.schema.response.pagination import PaginationBase
 
+from ...db.models.recipe import (
+    IngredientFoodModel,
+    RecipeComment,
+    RecipeIngredientModel,
+    RecipeInstruction,
+    RecipeModel,
+)
+from ..getter_dict import ExtrasGetterDict
 from .recipe_asset import RecipeAsset
 from .recipe_comments import RecipeCommentOut
 from .recipe_notes import RecipeNote
@@ -147,16 +155,7 @@ class Recipe(RecipeSummary):
 
     class Config:
         orm_mode = True
-
+        getter_dict = ExtrasGetterDict
-        @classmethod
-        def getter_dict(cls, name_orm: RecipeModel):
-            return {
-                **GetterDict(name_orm),
-                # "recipe_ingredient": [x.note for x in name_orm.recipe_ingredient],
-                # "recipe_category": [x.name for x in name_orm.recipe_category],
-                # "tags": [x.name for x in name_orm.tags],
-                "extras": {x.key_name: x.value for x in name_orm.extras},
-            }
 
     @validator("slug", always=True, pre=True, allow_reuse=True)
     def validate_slug(slug: str, values):  # type: ignore
@@ -199,6 +198,29 @@ class Recipe(RecipeSummary):
             return uuid4()
         return user_id
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [
+            selectinload(RecipeModel.assets),
+            selectinload(RecipeModel.comments).joinedload(RecipeComment.user),
+            selectinload(RecipeModel.extras),
+            joinedload(RecipeModel.recipe_category),
+            selectinload(RecipeModel.tags),
+            selectinload(RecipeModel.tools),
+            selectinload(RecipeModel.recipe_ingredient).joinedload(RecipeIngredientModel.unit),
+            selectinload(RecipeModel.recipe_ingredient)
+            .joinedload(RecipeIngredientModel.food)
+            .joinedload(IngredientFoodModel.extras),
+            selectinload(RecipeModel.recipe_ingredient)
+            .joinedload(RecipeIngredientModel.food)
+            .joinedload(IngredientFoodModel.label),
+            selectinload(RecipeModel.recipe_instructions).joinedload(RecipeInstruction.ingredient_references),
+            joinedload(RecipeModel.nutrition),
+            joinedload(RecipeModel.settings),
+            # for whatever reason, joinedload can mess up the order here, so use selectinload just this once
+            selectinload(RecipeModel.notes),
+        ]
+
 
 class RecipeLastMade(BaseModel):
     timestamp: datetime.datetime

mealie/schema/recipe/recipe_category.py

@@ -1,6 +1,8 @@
 from pydantic import UUID4
-from pydantic.utils import GetterDict
+from sqlalchemy.orm import selectinload
+from sqlalchemy.orm.interfaces import LoaderOption
 
+from mealie.db.models.recipe import RecipeModel, Tag
 from mealie.schema._mealie import MealieModel
 
 
@@ -19,12 +21,6 @@ class CategoryBase(CategoryIn):
     class Config:
         orm_mode = True
 
-        @classmethod
-        def getter_dict(_cls, name_orm):
-            return {
-                **GetterDict(name_orm),
-            }
-
 
 class CategoryOut(CategoryBase):
     slug: str
@@ -62,7 +58,13 @@ class TagOut(TagSave):
 
 
 class RecipeTagResponse(RecipeCategoryResponse):
-    pass
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [
+            selectinload(Tag.recipes).joinedload(RecipeModel.recipe_category),
+            selectinload(Tag.recipes).joinedload(RecipeModel.tags),
+            selectinload(Tag.recipes).joinedload(RecipeModel.tools),
+        ]
 
 
 from mealie.schema.recipe.recipe import RecipeSummary  # noqa: E402

mealie/schema/recipe/recipe_comments.py

@@ -1,7 +1,10 @@
 from datetime import datetime
 
 from pydantic import UUID4
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm.interfaces import LoaderOption
 
+from mealie.db.models.recipe import RecipeComment
 from mealie.schema._mealie import MealieModel
 from mealie.schema.response.pagination import PaginationBase
 
@@ -40,6 +43,10 @@ class RecipeCommentOut(RecipeCommentCreate):
     class Config:
         orm_mode = True
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [joinedload(RecipeComment.user)]
+
 
 class RecipeCommentPagination(PaginationBase):
     items: list[RecipeCommentOut]

mealie/schema/recipe/recipe_ingredient.py

@@ -2,14 +2,16 @@ from __future__ import annotations
 
 import datetime
 import enum
-from typing import Any
 from uuid import UUID, uuid4
 
 from pydantic import UUID4, Field, validator
-from pydantic.utils import GetterDict
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm.interfaces import LoaderOption
 
+from mealie.db.models.recipe import IngredientFoodModel
 from mealie.schema._mealie import MealieModel
 from mealie.schema._mealie.types import NoneFloat
+from mealie.schema.getter_dict import ExtrasGetterDict
 from mealie.schema.response.pagination import PaginationBase
 
 INGREDIENT_QTY_PRECISION = 3
@@ -37,19 +39,12 @@ class IngredientFood(CreateIngredientFood):
     update_at: datetime.datetime | None
 
     class Config:
-        class _FoodGetter(GetterDict):
-            def get(self, key: Any, default: Any = None) -> Any:
-                # Transform extras into key-value dict
-                if key == "extras":
-                    value = super().get(key, default)
-                    return {x.key_name: x.value for x in value}
-
-                # Keep all other fields as they are
-                else:
-                    return super().get(key, default)
-
         orm_mode = True
-        getter_dict = _FoodGetter
+        getter_dict = ExtrasGetterDict
+
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [joinedload(IngredientFoodModel.extras), joinedload(IngredientFoodModel.label)]
 
 
 class IngredientFoodPagination(PaginationBase):

mealie/schema/recipe/recipe_share_token.py

@@ -1,9 +1,12 @@
 from datetime import datetime, timedelta
 
 from pydantic import UUID4, Field
+from sqlalchemy.orm import selectinload
+from sqlalchemy.orm.interfaces import LoaderOption
 
 from mealie.schema._mealie import MealieModel
 
+from ...db.models.recipe import RecipeIngredientModel, RecipeInstruction, RecipeModel, RecipeShareTokenModel
 from .recipe import Recipe
 
 
@@ -33,3 +36,26 @@ class RecipeShareToken(RecipeShareTokenSummary):
 
     class Config:
         orm_mode = True
+
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [
+            selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.recipe_category),
+            selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.tags),
+            selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.tools),
+            selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.nutrition),
+            selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.settings),
+            selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.assets),
+            selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.notes),
+            selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.extras),
+            selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.comments),
+            selectinload(RecipeShareTokenModel.recipe)
+            .joinedload(RecipeModel.recipe_instructions)
+            .joinedload(RecipeInstruction.ingredient_references),
+            selectinload(RecipeShareTokenModel.recipe)
+            .joinedload(RecipeModel.recipe_ingredient)
+            .joinedload(RecipeIngredientModel.unit),
+            selectinload(RecipeShareTokenModel.recipe)
+            .joinedload(RecipeModel.recipe_ingredient)
+            .joinedload(RecipeIngredientModel.food),
+        ]

mealie/schema/recipe/recipe_tool.py

@@ -1,7 +1,11 @@
 from pydantic import UUID4
+from sqlalchemy.orm import selectinload
+from sqlalchemy.orm.interfaces import LoaderOption
 
 from mealie.schema._mealie import MealieModel
 
+from ...db.models.recipe import RecipeModel, Tool
+
 
 class RecipeToolCreate(MealieModel):
     name: str
@@ -21,12 +25,20 @@ class RecipeToolOut(RecipeToolCreate):
 
 
 class RecipeToolResponse(RecipeToolOut):
-    recipes: list["Recipe"] = []
+    recipes: list["RecipeSummary"] = []
 
     class Config:
         orm_mode = True
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [
+            selectinload(Tool.recipes).joinedload(RecipeModel.recipe_category),
+            selectinload(Tool.recipes).joinedload(RecipeModel.tags),
+            selectinload(Tool.recipes).joinedload(RecipeModel.tools),
+        ]
 
-from .recipe import Recipe  # noqa: E402
+
+from .recipe import RecipeSummary  # noqa: E402
 
 RecipeToolResponse.update_forward_refs()

mealie/schema/reports/reports.py

@@ -3,7 +3,10 @@ import enum
 
 from pydantic import Field
 from pydantic.types import UUID4
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm.interfaces import LoaderOption
 
+from mealie.db.models.group import ReportModel
 from mealie.schema._mealie import MealieModel
 
 
@@ -53,3 +56,7 @@ class ReportOut(ReportSummary):
 
     class Config:
         orm_mode = True
+
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [joinedload(ReportModel.entries)]

mealie/schema/user/user.py

@@ -5,7 +5,8 @@ from uuid import UUID
 
 from pydantic import UUID4, Field, validator
 from pydantic.types import constr
-from pydantic.utils import GetterDict
+from sqlalchemy.orm import joinedload, selectinload
+from sqlalchemy.orm.interfaces import LoaderOption
 
 from mealie.core.config import get_app_dirs, get_app_settings
 from mealie.db.models.users import User
@@ -15,6 +16,9 @@ from mealie.schema.group.group_preferences import ReadGroupPreferences
 from mealie.schema.recipe import RecipeSummary
 from mealie.schema.response.pagination import PaginationBase
 
+from ...db.models.group import Group
+from ...db.models.recipe import RecipeModel
+from ..getter_dict import GroupGetterDict, UserGetterDict
 from ..recipe import CategoryBase
 
 DEFAULT_INTEGRATION_ID = "generic"
@@ -78,19 +82,8 @@ class UserBase(MealieModel):
     can_organize: bool = False
 
     class Config:
-        class _UserGetter(GetterDict):
-            def get(self, key: Any, default: Any = None) -> Any:
-                # Transform extras into key-value dict
-                if key == "group":
-                    value = super().get(key, default)
-                    return value.group.name
-
-                # Keep all other fields as they are
-                else:
-                    return super().get(key, default)
-
         orm_mode = True
-        getter_dict = _UserGetter
+        getter_dict = GroupGetterDict
 
         schema_extra = {
             "example": {
@@ -118,13 +111,11 @@ class UserOut(UserBase):
     class Config:
         orm_mode = True
 
-        @classmethod
+        getter_dict = UserGetterDict
-        def getter_dict(cls, ormModel: User):
+
-            return {
+    @classmethod
-                **GetterDict(ormModel),
+    def loader_options(cls) -> list[LoaderOption]:
-                "group": ormModel.group.name,
+        return [joinedload(User.group), joinedload(User.favorite_recipes), joinedload(User.tokens)]
-                "favorite_recipes": [x.slug for x in ormModel.favorite_recipes],
-            }
 
 
 class UserPagination(PaginationBase):
@@ -136,13 +127,16 @@ class UserFavorites(UserBase):
 
     class Config:
         orm_mode = True
+        getter_dict = GroupGetterDict
 
-        @classmethod
+    @classmethod
-        def getter_dict(cls, ormModel: User):
+    def loader_options(cls) -> list[LoaderOption]:
-            return {
+        return [
-                **GetterDict(ormModel),
+            joinedload(User.group),
-                "group": ormModel.group.name,
+            selectinload(User.favorite_recipes).joinedload(RecipeModel.recipe_category),
-            }
+            selectinload(User.favorite_recipes).joinedload(RecipeModel.tags),
+            selectinload(User.favorite_recipes).joinedload(RecipeModel.tools),
+        ]
 
 
 class PrivateUser(UserOut):
@@ -175,6 +169,10 @@ class PrivateUser(UserOut):
     def directory(self) -> Path:
         return PrivateUser.get_directory(self.id)
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [joinedload(User.group), selectinload(User.favorite_recipes), joinedload(User.tokens)]
+
 
 class UpdateGroup(GroupBase):
     id: UUID4
@@ -211,6 +209,17 @@ class GroupInDB(UpdateGroup):
     def exports(self) -> Path:
         return GroupInDB.get_export_directory(self.id)
 
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [
+            joinedload(Group.categories),
+            joinedload(Group.webhooks),
+            joinedload(Group.preferences),
+            selectinload(Group.users).joinedload(User.group),
+            selectinload(Group.users).joinedload(User.favorite_recipes),
+            selectinload(Group.users).joinedload(User.tokens),
+        ]
+
 
 class GroupPagination(PaginationBase):
     items: list[GroupInDB]

mealie/schema/user/user_passwords.py

@@ -1,7 +1,10 @@
 from pydantic import UUID4
+from sqlalchemy.orm import selectinload
+from sqlalchemy.orm.interfaces import LoaderOption
 
 from mealie.schema._mealie import MealieModel
 
+from ...db.models.users import PasswordResetModel, User
 from .user import PrivateUser
 
 
@@ -33,3 +36,11 @@ class PrivatePasswordResetToken(SavePasswordResetToken):
 
     class Config:
         orm_mode = True
+
+    @classmethod
+    def loader_options(cls) -> list[LoaderOption]:
+        return [
+            selectinload(PasswordResetModel.user).joinedload(User.group),
+            selectinload(PasswordResetModel.user).joinedload(User.favorite_recipes),
+            selectinload(PasswordResetModel.user).joinedload(User.tokens),
+        ]