feat: add support for API extras on shopping lists, shopping list items, and food data (#1619)

* added api extras to other tables
genericized api extras model from recipes
added extras column to ingredient foods
added extras column to shopping lists
added extras column to shopping list items

* updated alembic version test

* made mypy happy

* added TODO on test that does nothing

* added extras tests for lists, items, and foods

* added docs for new extras

* modified alembic versions to eliminate branching

alembic/versions/2022-08-29-13.57.40_44e8d670719d_add_extras_to_shopping_lists_list_items_.py

@@ -0,0 +1,72 @@
+"""add extras to shopping lists, list items, and ingredient foods
+
+Revision ID: 44e8d670719d
+Revises: 188374910655
+Create Date: 2022-08-29 13:57:40.452245
+
+"""
+import sqlalchemy as sa
+
+import mealie.db.migration_types
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "44e8d670719d"
+down_revision = "089bfa50d0ed"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "shopping_list_extras",
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.Column("update_at", sa.DateTime(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("key_name", sa.String(), nullable=True),
+        sa.Column("value", sa.String(), nullable=True),
+        sa.Column("shopping_list_id", mealie.db.migration_types.GUID(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["shopping_list_id"],
+            ["shopping_lists.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "ingredient_food_extras",
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.Column("update_at", sa.DateTime(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("key_name", sa.String(), nullable=True),
+        sa.Column("value", sa.String(), nullable=True),
+        sa.Column("ingredient_food_id", mealie.db.migration_types.GUID(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["ingredient_food_id"],
+            ["ingredient_foods.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "shopping_list_item_extras",
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.Column("update_at", sa.DateTime(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("key_name", sa.String(), nullable=True),
+        sa.Column("value", sa.String(), nullable=True),
+        sa.Column("shopping_list_item_id", mealie.db.migration_types.GUID(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["shopping_list_item_id"],
+            ["shopping_list_items.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("shopping_list_item_extras")
+    op.drop_table("ingredient_food_extras")
+    op.drop_table("shopping_list_extras")
+    # ### end Alembic commands ###

docs/docs/documentation/getting-started/api-usage.md

@@ -10,11 +10,26 @@ Mealie supports long-live api tokens in the user frontend. See [user settings pa
 ### Exploring Your Local API
 On your local installation you can access interactive API documentation that provides `curl` examples and expected results. This allows you to easily test and interact with your API to identify places to include your own functionality. You can visit the documentation at `http://mealie.yourdomain.com/docs` or see the example at the [Demo Site](https://mealie-demo.hay-kot.dev/docs)
 
-### Recipe Extras
+### Extras
+#### Recipe Extras
 Recipes extras are a key feature of the Mealie API. They allow you to create custom json key/value pairs within a recipe to reference from 3rd part applications. You can use these keys to contain information to trigger automation or custom messages to relay to your desired device.
 
 For example you could add `{"message": "Remember to thaw the chicken"}` to a recipe and use the webhooks built into mealie to send that message payload to a destination to be processed.
 
+#### Shopping List and Food Extras
+Similarly to recipes, extras are supported on shopping lists, shopping list items, and foods. At this time they are only accessible through the API. Extras for these objects allow for rich integrations between the Mealie shopping list and your favorite list manager, such as Alexa, ToDoist, Trello, or any other list manager with an API.
+
+To keep shopping lists in sync, for instance, you can store your Trello list id on your Mealie shopping list: <br />
+`{"trello_list_id": "5abbe4b7ddc1b351ef961414"}`
+
+Now if an update is made to your shopping list, you know which Trello list also needs to be updated. Similarly, you can also keep track of individual cards on your Trello list by storing data on shopping list items: <br />
+`{"trello_card_id": "bab414bde415cd715efb9361"}`
+
+Sometimes you may want to exclude certain foods from syncing to your external list, such as water, so you can add a custom property to your "water" food: <br />
+`{"trello_exclude_food": "true"}`
+
+You can combine your custom data definitions with our Event Subscriptions API, enabling you to keep your external list up-to-date in real time.
+
 ### Pagination and Filtering
 Most document types share a uniform pagination and filtering API (e.g. `GET /api/recipes`). These allow you to filter by an arbitrary combination of criteria and return only a certain number of documents (i.e. a single "page" of documents).
 

mealie/db/models/group/shopping_list.py

@@ -2,6 +2,7 @@ from sqlalchemy import Boolean, Column, Float, ForeignKey, Integer, String, orm
 from sqlalchemy.ext.orderinglist import ordering_list
 
 from mealie.db.models.labels import MultiPurposeLabel
+from mealie.db.models.recipe.api_extras import ShoppingListExtras, ShoppingListItemExtras, api_extras
 
 from .._model_base import BaseMixins, SqlAlchemyBase
 from .._model_utils import GUID, auto_init
@@ -38,6 +39,7 @@ class ShoppingListItem(SqlAlchemyBase, BaseMixins):
     note = Column(String)
 
     is_food = Column(Boolean, default=False)
+    extras: list[ShoppingListItemExtras] = orm.relationship("ShoppingListItemExtras", cascade="all, delete-orphan")
 
     # Scaling Items
     unit_id = Column(GUID, ForeignKey("ingredient_units.id"))
@@ -55,6 +57,7 @@ class ShoppingListItem(SqlAlchemyBase, BaseMixins):
     class Config:
         exclude = {"id", "label", "food", "unit"}
 
+    @api_extras
     @auto_init()
     def __init__(self, **_) -> None:
         pass
@@ -95,10 +98,12 @@ class ShoppingList(SqlAlchemyBase, BaseMixins):
     )
 
     recipe_references = orm.relationship(ShoppingListRecipeReference, cascade="all, delete, delete-orphan")
+    extras: list[ShoppingListExtras] = orm.relationship("ShoppingListExtras", cascade="all, delete-orphan")
 
     class Config:
         exclude = {"id", "list_items"}
 
+    @api_extras
     @auto_init()
     def __init__(self, **_) -> None:
         pass

mealie/db/models/recipe/api_extras.py

@@ -4,13 +4,54 @@ from mealie.db.models._model_base import SqlAlchemyBase
 from mealie.db.models._model_utils.guid import GUID
 
 
-class ApiExtras(SqlAlchemyBase):
-    __tablename__ = "api_extras"
+def api_extras(func):
+    """Decorator function to unpack the extras into a dict; requires an "extras" column"""
+
+    def wrapper(*args, **kwargs):
+        extras = kwargs.pop("extras")
+
+        if extras is None:
+            extras = []
+        else:
+            extras = [{"key": key, "value": value} for key, value in extras.items()]
+
+        return func(*args, extras=extras, **kwargs)
+
+    return wrapper
+
+
+class ExtrasGeneric:
+    """
+    Template for API extensions
+
+    This class is not an actual table, so it does not inherit from SqlAlchemyBase
+    """
+
     id = sa.Column(sa.Integer, primary_key=True)
-    recipee_id = sa.Column(GUID, sa.ForeignKey("recipes.id"))
     key_name = sa.Column(sa.String)
     value = sa.Column(sa.String)
 
     def __init__(self, key, value) -> None:
         self.key_name = key
         self.value = value
+
+
+# used specifically for recipe extras
+class ApiExtras(ExtrasGeneric, SqlAlchemyBase):
+    __tablename__ = "api_extras"
+    recipee_id = sa.Column(GUID, sa.ForeignKey("recipes.id"))
+
+
+class IngredientFoodExtras(ExtrasGeneric, SqlAlchemyBase):
+    __tablename__ = "ingredient_food_extras"
+    ingredient_food_id = sa.Column(GUID, sa.ForeignKey("ingredient_foods.id"))
+
+
+class ShoppingListExtras(ExtrasGeneric, SqlAlchemyBase):
+    __tablename__ = "shopping_list_extras"
+    shopping_list_id = sa.Column(GUID, sa.ForeignKey("shopping_lists.id"))
+
+
+class ShoppingListItemExtras(ExtrasGeneric, SqlAlchemyBase):
+    __tablename__ = "shopping_list_item_extras"
+    shopping_list_item_id = sa.Column(GUID, sa.ForeignKey("shopping_list_items.id"))

mealie/db/models/recipe/ingredient.py

@@ -2,6 +2,7 @@ from sqlalchemy import Boolean, Column, Float, ForeignKey, Integer, String, orm
 
 from mealie.db.models._model_base import BaseMixins, SqlAlchemyBase
 from mealie.db.models.labels import MultiPurposeLabel
+from mealie.db.models.recipe.api_extras import IngredientFoodExtras, api_extras
 
 from .._model_utils import auto_init
 from .._model_utils.guid import GUID
@@ -38,10 +39,12 @@ class IngredientFoodModel(SqlAlchemyBase, BaseMixins):
     name = Column(String)
     description = Column(String)
     ingredients = orm.relationship("RecipeIngredient", back_populates="food")
+    extras: list[IngredientFoodExtras] = orm.relationship("IngredientFoodExtras", cascade="all, delete-orphan")
 
     label_id = Column(GUID, ForeignKey("multi_purpose_labels.id"))
     label = orm.relationship(MultiPurposeLabel, uselist=False, back_populates="foods")
 
+    @api_extras
     @auto_init()
     def __init__(self, **_) -> None:
         pass

mealie/db/models/recipe/recipe.py

@@ -10,7 +10,7 @@ from mealie.db.models._model_utils.guid import GUID
 from .._model_base import BaseMixins, SqlAlchemyBase
 from .._model_utils import auto_init
 from ..users.user_to_favorite import users_to_favorites
-from .api_extras import ApiExtras
+from .api_extras import ApiExtras, api_extras
 from .assets import RecipeAsset
 from .category import recipes_to_categories
 from .comment import RecipeComment
@@ -24,21 +24,6 @@ from .tag import recipes_to_tags
 from .tool import recipes_to_tools
 
 
-# Decorator function to unpack the extras into a dict
-def recipe_extras(func):
-    def wrapper(*args, **kwargs):
-        extras = kwargs.pop("extras")
-
-        if extras is None:
-            extras = []
-        else:
-            extras = [{"key": key, "value": value} for key, value in extras.items()]
-
-        return func(*args, extras=extras, **kwargs)
-
-    return wrapper
-
-
 class RecipeModel(SqlAlchemyBase, BaseMixins):
     __tablename__ = "recipes"
     __table_args__ = (sa.UniqueConstraint("slug", "group_id", name="recipe_slug_group_id_key"),)
@@ -139,7 +124,7 @@ class RecipeModel(SqlAlchemyBase, BaseMixins):
         assert name != ""
         return name
 
-    @recipe_extras
+    @api_extras
     @auto_init()
     def __init__(
         self,

mealie/schema/group/group_shopping_list.py

@@ -4,7 +4,9 @@ from datetime import datetime
 from typing import Optional, Union
 
 from pydantic import UUID4
+from pydantic.utils import GetterDict
 
+from mealie.db.models.group.shopping_list import ShoppingList, ShoppingListItem
 from mealie.schema._mealie import MealieModel
 from mealie.schema.recipe.recipe_ingredient import IngredientFood, IngredientUnit
 from mealie.schema.response.pagination import PaginationBase
@@ -39,6 +41,7 @@ class ShoppingListItemCreate(MealieModel):
 
     label_id: Optional[UUID4] = None
     recipe_references: list[ShoppingListItemRecipeRef] = []
+    extras: Optional[dict] = {}
 
     created_at: Optional[datetime]
     update_at: Optional[datetime]
@@ -55,9 +58,17 @@ class ShoppingListItemOut(ShoppingListItemUpdate):
     class Config:
         orm_mode = True
 
+        @classmethod
+        def getter_dict(cls, name_orm: ShoppingListItem):
+            return {
+                **GetterDict(name_orm),
+                "extras": {x.key_name: x.value for x in name_orm.extras},
+            }
+
 
 class ShoppingListCreate(MealieModel):
     name: str = None
+    extras: Optional[dict] = {}
 
     created_at: Optional[datetime]
     update_at: Optional[datetime]
@@ -84,6 +95,13 @@ class ShoppingListSummary(ShoppingListSave):
     class Config:
         orm_mode = True
 
+        @classmethod
+        def getter_dict(cls, name_orm: ShoppingList):
+            return {
+                **GetterDict(name_orm),
+                "extras": {x.key_name: x.value for x in name_orm.extras},
+            }
+
 
 class ShoppingListPagination(PaginationBase):
     items: list[ShoppingListSummary]

mealie/schema/recipe/recipe_ingredient.py

@@ -6,7 +6,9 @@ from typing import Optional, Union
 from uuid import UUID, uuid4
 
 from pydantic import UUID4, Field, validator
+from pydantic.utils import GetterDict
 
+from mealie.db.models.recipe.ingredient import IngredientFoodModel
 from mealie.schema._mealie import MealieModel
 from mealie.schema._mealie.types import NoneFloat
 from mealie.schema.response.pagination import PaginationBase
@@ -15,6 +17,7 @@ from mealie.schema.response.pagination import PaginationBase
 class UnitFoodBase(MealieModel):
     name: str
     description: str = ""
+    extras: Optional[dict] = {}
 
 
 class CreateIngredientFood(UnitFoodBase):
@@ -34,6 +37,13 @@ class IngredientFood(CreateIngredientFood):
     class Config:
         orm_mode = True
 
+        @classmethod
+        def getter_dict(cls, name_orm: IngredientFoodModel):
+            return {
+                **GetterDict(name_orm),
+                "extras": {x.key_name: x.value for x in name_orm.extras},
+            }
+
 
 class IngredientFoodPagination(PaginationBase):
     items: list[IngredientFood]

tests/integration_tests/user_group_tests/test_group_shopping_list_items.py

@@ -1,6 +1,7 @@
 import random
 from uuid import uuid4
 
+import pytest
 from fastapi.testclient import TestClient
 from pydantic import UUID4
 
@@ -14,10 +15,12 @@ class Routes:
     shopping = "/api/groups/shopping"
     items = shopping + "/items"
 
-    def item(item_id: str) -> str:
+    @staticmethod
+    def item(item_id: str | UUID4) -> str:
         return f"{Routes.items}/{item_id}"
 
-    def shopping_list(list_id: str) -> str:
+    @staticmethod
+    def shopping_list(list_id: str | UUID4) -> str:
         return f"{Routes.shopping}/lists/{list_id}"
 
 
@@ -162,9 +165,9 @@ def test_shopping_list_items_update_many_reorder(
     response = api_client.get(Routes.shopping_list(list_with_items.id), headers=unique_user.token)
     response_list = utils.assert_derserialize(response, 200)
 
-    for i, item in enumerate(response_list["listItems"]):
-        assert item["position"] == i
-        assert item["id"] == str(list_items[i].id)
+    for i, item_data in enumerate(response_list["listItems"]):
+        assert item_data["position"] == i
+        assert item_data["id"] == str(list_items[i].id)
 
 
 def test_shopping_list_items_update_many_consolidates_common_items(
@@ -194,6 +197,7 @@ def test_shopping_list_items_update_many_consolidates_common_items(
     assert response_list["listItems"][0]["note"] == master_note
 
 
+@pytest.mark.skip("TODO: Implement")
 def test_shopping_list_items_update_many_remove_recipe_with_other_items(
     api_client: TestClient,
     unique_user: TestUser,
@@ -201,3 +205,38 @@ def test_shopping_list_items_update_many_remove_recipe_with_other_items(
 ) -> None:
     # list_items = list_with_items.list_items
     pass
+
+
+def test_shopping_list_item_extras(
+    api_client: TestClient, unique_user: TestUser, shopping_list: ShoppingListOut
+) -> None:
+    key_str_1 = random_string()
+    val_str_1 = random_string()
+
+    key_str_2 = random_string()
+    val_str_2 = random_string()
+
+    # create an item with extras
+    new_item_data = create_item(shopping_list.id)
+    new_item_data["extras"] = {key_str_1: val_str_1}
+
+    response = api_client.post(Routes.items, json=new_item_data, headers=unique_user.token)
+    item_as_json = utils.assert_derserialize(response, 201)
+
+    # make sure the extra persists
+    extras = item_as_json["extras"]
+    assert key_str_1 in extras
+    assert extras[key_str_1] == val_str_1
+
+    # add more extras to the item
+    item_as_json["extras"][key_str_2] = val_str_2
+
+    response = api_client.put(Routes.item(item_as_json["id"]), json=item_as_json, headers=unique_user.token)
+    item_as_json = utils.assert_derserialize(response, 200)
+
+    # make sure both the new extra and original extra persist
+    extras = item_as_json["extras"]
+    assert key_str_1 in extras
+    assert key_str_2 in extras
+    assert extras[key_str_1] == val_str_1
+    assert extras[key_str_2] == val_str_2

tests/integration_tests/user_group_tests/test_group_shopping_lists.py

@@ -1,6 +1,7 @@
 import random
 
 from fastapi.testclient import TestClient
+from pydantic import UUID4
 
 from mealie.schema.group.group_shopping_list import ShoppingListOut
 from mealie.schema.recipe.recipe import Recipe
@@ -12,17 +13,19 @@ from tests.utils.fixture_schemas import TestUser
 class Routes:
     base = "/api/groups/shopping/lists"
 
-    def item(item_id: str) -> str:
+    @staticmethod
+    def item(item_id: str | UUID4) -> str:
         return f"{Routes.base}/{item_id}"
 
-    def add_recipe(item_id: str, recipe_id: str) -> str:
+    @staticmethod
+    def add_recipe(item_id: str | UUID4, recipe_id: str | UUID4) -> str:
         return f"{Routes.item(item_id)}/recipe/{recipe_id}"
 
 
 def test_shopping_lists_get_all(api_client: TestClient, unique_user: TestUser, shopping_lists: list[ShoppingListOut]):
-    all_lists = api_client.get(Routes.base, headers=unique_user.token)
-    assert all_lists.status_code == 200
-    all_lists = all_lists.json()["items"]
+    response = api_client.get(Routes.base, headers=unique_user.token)
+    assert response.status_code == 200
+    all_lists = response.json()["items"]
 
     assert len(all_lists) == len(shopping_lists)
 
@@ -199,3 +202,39 @@ def test_shopping_lists_remove_recipe_multiple_quantity(
     refs = as_json["recipeReferences"]
     assert len(refs) == 1
     assert refs[0]["recipeId"] == str(recipe.id)
+
+
+def test_shopping_list_extras(
+    api_client: TestClient,
+    unique_user: TestUser,
+):
+    key_str_1 = random_string()
+    val_str_1 = random_string()
+
+    key_str_2 = random_string()
+    val_str_2 = random_string()
+
+    # create a list with extras
+    new_list_data: dict = {"name": random_string()}
+    new_list_data["extras"] = {key_str_1: val_str_1}
+
+    response = api_client.post(Routes.base, json=new_list_data, headers=unique_user.token)
+    list_as_json = utils.assert_derserialize(response, 201)
+
+    # make sure the extra persists
+    extras = list_as_json["extras"]
+    assert key_str_1 in extras
+    assert extras[key_str_1] == val_str_1
+
+    # add more extras to the list
+    list_as_json["extras"][key_str_2] = val_str_2
+
+    response = api_client.put(Routes.item(list_as_json["id"]), json=list_as_json, headers=unique_user.token)
+    list_as_json = utils.assert_derserialize(response, 200)
+
+    # make sure both the new extra and original extra persist
+    extras = list_as_json["extras"]
+    assert key_str_1 in extras
+    assert key_str_2 in extras
+    assert extras[key_str_1] == val_str_1
+    assert extras[key_str_2] == val_str_2

tests/integration_tests/user_recipe_tests/test_recipe_foods.py

@@ -1,7 +1,10 @@
+from collections.abc import Generator
+
 import pytest
 from fastapi.testclient import TestClient
 
 from mealie.schema.recipe.recipe_ingredient import CreateIngredientFood
+from tests import utils
 from tests.utils.factories import random_string
 from tests.utils.fixture_schemas import TestUser
 
@@ -9,12 +12,13 @@ from tests.utils.fixture_schemas import TestUser
 class Routes:
     base = "/api/foods"
 
+    @staticmethod
     def item(item_id: int) -> str:
         return f"{Routes.base}/{item_id}"
 
 
 @pytest.fixture(scope="function")
-def food(api_client: TestClient, unique_user: TestUser) -> dict:
+def food(api_client: TestClient, unique_user: TestUser) -> Generator[dict, None, None]:
     data = CreateIngredientFood(
         name=random_string(10),
         description=random_string(10),
@@ -74,3 +78,39 @@ def test_delete_food(api_client: TestClient, food: dict, unique_user: TestUser):
 
     response = api_client.get(Routes.item(id), headers=unique_user.token)
     assert response.status_code == 404
+
+
+def test_food_extras(
+    api_client: TestClient,
+    unique_user: TestUser,
+):
+    key_str_1 = random_string()
+    val_str_1 = random_string()
+
+    key_str_2 = random_string()
+    val_str_2 = random_string()
+
+    # create a food with extras
+    new_food_data: dict = {"name": random_string()}
+    new_food_data["extras"] = {key_str_1: val_str_1}
+
+    response = api_client.post(Routes.base, json=new_food_data, headers=unique_user.token)
+    food_as_json = utils.assert_derserialize(response, 201)
+
+    # make sure the extra persists
+    extras = food_as_json["extras"]
+    assert key_str_1 in extras
+    assert extras[key_str_1] == val_str_1
+
+    # add more extras to the food
+    food_as_json["extras"][key_str_2] = val_str_2
+
+    response = api_client.put(Routes.item(food_as_json["id"]), json=food_as_json, headers=unique_user.token)
+    food_as_json = utils.assert_derserialize(response, 200)
+
+    # make sure both the new extra and original extra persist
+    extras = food_as_json["extras"]
+    assert key_str_1 in extras
+    assert key_str_2 in extras
+    assert extras[key_str_1] == val_str_1
+    assert extras[key_str_2] == val_str_2

tests/unit_tests/services_tests/backup_v2_tests/test_alchemy_exporter.py

@@ -4,7 +4,7 @@ from mealie.core.config import get_app_settings
 from mealie.services.backups_v2.alchemy_exporter import AlchemyExporter
 
 ALEMBIC_VERSIONS = [
-    {"version_num": "089bfa50d0ed"},
+    {"version_num": "44e8d670719d"},
 ]