Merge pull request #3170 from michael-genson/fix/make-mealie-alpha-migrations-more-fault-tolerant

fix: Make Mealie Alpha Migrations More Fault Tolerant

mealie/services/migrations/_migration_base.py

@@ -14,6 +14,7 @@ from mealie.schema.reports.reports import (
     ReportCategory,
     ReportCreate,
     ReportEntryCreate,
+    ReportEntryOut,
     ReportOut,
     ReportSummary,
     ReportSummaryStatus,
@@ -91,6 +92,7 @@ class BaseMigrator(BaseService):
         is_success = True
         is_failure = True
 
+        new_entries: list[ReportEntryOut] = []
         for entry in self.report_entries:
             if is_failure and entry.success:
                 is_failure = False
@@ -98,7 +100,7 @@ class BaseMigrator(BaseService):
             if is_success and not entry.success:
                 is_success = False
 
-            self.db.group_report_entries.create(entry)
+            new_entries.append(self.db.group_report_entries.create(entry))
 
         if is_success:
             self.report.status = ReportSummaryStatus.success
@@ -109,6 +111,7 @@ class BaseMigrator(BaseService):
         if not is_success and not is_failure:
             self.report.status = ReportSummaryStatus.partial
 
+        self.report.entries = new_entries
         self.db.group_reports.update(self.report.id, self.report)
 
     def migrate(self, report_name: str) -> ReportSummary:

mealie/services/migrations/mealie_alpha.py

@@ -5,6 +5,7 @@ import zipfile
 from pathlib import Path
 
 from mealie.schema.recipe.recipe import Recipe
+from mealie.schema.reports.reports import ReportEntryCreate
 
 from ._migration_base import BaseMigrator
 from .utils.migration_alias import MigrationAlias
@@ -55,20 +56,28 @@ class MealieAlphaMigrator(BaseMigrator):
                 zip_file.extractall(tmpdir)
 
             temp_path = Path(tmpdir)
-
             recipe_lookup: dict[str, Path] = {}
-            recipes_as_dicts = []
 
-            for x in temp_path.rglob("**/recipes/**/[!.]*.json"):
+            recipes: list[Recipe] = []
-                if (y := MigrationReaders.json(x)) is not None:
+            for recipe_json_path in temp_path.rglob("**/recipes/**/[!.]*.json"):
-                    recipes_as_dicts.append(y)
+                try:
-                    slug = y["slug"]
+                    if (recipe_as_dict := MigrationReaders.json(recipe_json_path)) is not None:
-                    recipe_lookup[slug] = x.parent
+                        recipe = self._convert_to_new_schema(recipe_as_dict)
-
+                        recipes.append(recipe)
-            recipes = [self._convert_to_new_schema(x) for x in recipes_as_dicts]
+                        slug = recipe_as_dict["slug"]
+                        recipe_lookup[slug] = recipe_json_path.parent
+                except Exception as e:
+                    self.logger.exception(e)
+                    self.report_entries.append(
+                        ReportEntryCreate(
+                            report_id=self.report_id,
+                            success=False,
+                            message=f"Failed to import {recipe_json_path.name}",
+                            exception=f"{e.__class__.__name__}: {e}",
+                        )
+                    )
 
             results = self.import_recipes_to_database(recipes)
-
             for slug, recipe_id, status in results:
                 if not status:
                     continue

tests/integration_tests/recipe_migration_tests/test_recipe_migrations.py

@@ -1,10 +1,14 @@
+import os
 from dataclasses import dataclass
 from pathlib import Path
+from tempfile import TemporaryDirectory
+from zipfile import ZipFile
 
 import pytest
 from fastapi.testclient import TestClient
 
 from mealie.schema.group.group_migration import SupportedMigrations
+from mealie.schema.reports.reports import ReportEntryOut
 from tests import data as test_data
 from tests.utils import api_routes
 from tests.utils.assertion_helpers import assert_derserialize
@@ -60,8 +64,10 @@ def test_recipe_migration(api_client: TestClient, unique_user: TestUser, mig: Mi
     response = api_client.get(api_routes.groups_reports_item_id(report_id), headers=unique_user.token)
 
     assert response.status_code == 200
+    response_json = response.json()
+    assert response_json["entries"]
 
-    for item in response.json()["entries"]:
+    for item in response_json["entries"]:
         assert item["success"]
 
     # Validate Create Event
@@ -77,3 +83,63 @@ def test_recipe_migration(api_client: TestClient, unique_user: TestUser, mig: Mi
     query_data = assert_derserialize(response)
     events = query_data["items"]
     assert len(events)
+
+
+def test_bad_mealie_alpha_data_is_ignored(api_client: TestClient, unique_user: TestUser):
+    with TemporaryDirectory() as tmpdir:
+        with ZipFile(test_data.migrations_mealie) as zf:
+            zf.extractall(tmpdir)
+
+        invalid_recipe_dir = os.path.join(tmpdir, "mealie_2021-Dec-08", "recipes", "invalid-recipe")
+        os.makedirs(invalid_recipe_dir, exist_ok=True)
+        invalid_json_path = os.path.join(invalid_recipe_dir, "invalid-recipe.json")
+        try:
+            with open(invalid_json_path, "w"):
+                pass  # write nothing to the file, which is invalid JSON
+        except Exception:
+            raise Exception(os.listdir(tmpdir))
+
+        modified_test_data = os.path.join(tmpdir, "modified-test-data.zip")
+        with ZipFile(modified_test_data, "w") as zf:
+            for root, _, files in os.walk(tmpdir):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    zf.write(file_path, arcname=os.path.relpath(file_path, tmpdir))
+
+        payload = {
+            "migration_type": SupportedMigrations.mealie_alpha.value,
+        }
+
+        file_payload = {
+            "archive": Path(modified_test_data).read_bytes(),
+        }
+
+        response = api_client.post(
+            api_routes.groups_migrations, data=payload, files=file_payload, headers=unique_user.token
+        )
+
+        assert response.status_code == 200
+        report_id = response.json()["id"]
+
+    # Validate Results
+    response = api_client.get(api_routes.groups_reports_item_id(report_id), headers=unique_user.token)
+
+    assert response.status_code == 200
+    response_json = response.json()
+    assert response_json["entries"]
+
+    failed_item = None
+    failed_item_count = 0
+    for item in response_json["entries"]:
+        if item["success"]:
+            continue
+
+        failed_item = item
+        failed_item_count += 1
+
+    assert failed_item
+    assert failed_item_count == 1
+
+    report_entry = ReportEntryOut.model_validate(failed_item)
+    assert report_entry.message == "Failed to import invalid-recipe.json"
+    assert report_entry.exception == "JSONDecodeError: Expecting value: line 1 column 1 (char 0)"