feat: add reports to bulk recipe import (url) (#1294)

* remove unused docker and caddy configs

* add experimental nested configs

* switch to nest under docker-compose

* remove v-card

* bulk parser backend re-implementation

* refactor UI for bulk importer

* remove migration specific report text

.vscode/settings.json

@@ -46,5 +46,12 @@
   "python.linting.mypyEnabled": true,
   "python.sortImports.path": "${workspaceFolder}/.venv/bin/isort",
   "search.mode": "reuseEditor",
-  "python.testing.unittestArgs": ["-v", "-s", "./tests", "-p", "test_*.py"]
+  "python.testing.unittestArgs": ["-v", "-s", "./tests", "-p", "test_*.py"],
+  "explorer.fileNesting.enabled": true,
+  "explorer.fileNesting.patterns": {
+    "package.json": "package-lock.json, yarn.lock, .eslintrc.js, tsconfig.json, .prettierrc, .editorconfig",
+    "pyproject.toml": "poetry.lock, alembic.ini, .pylintrc, .flake8",
+    "netlify.toml": "runtime.txt",
+    "docker-compose.yml": "Dockerfile, .dockerignore, docker-compose.dev.yml, docker-compose.yml"
+  }
 }

docs/Caddyfile

@@ -1,15 +0,0 @@
-{
-  auto_https off
-}
-
-:80 {
-  root * /srv
-  encode gzip
-  uri strip_suffix /
-
-  handle {
-    try_files {path} {path}/ /index.html
-    file_server 
-  }
-
-}

docs/Dockerfile

@@ -1,10 +0,0 @@
-FROM python:3.8-slim as build-stage
-WORKDIR /app
-RUN pip install --no-cache-dir mkdocs mkdocs-material
-COPY . .
-RUN mkdocs build
-
-FROM caddy:alpine
-WORKDIR /app
-COPY ./Caddyfile /etc/caddy/Caddyfile
-COPY --from=build-stage /app/site /srv

docs/docker-compose.yml

@@ -1,11 +0,0 @@
-version: "3"
-services:
-  wiki:
-    container_name: mealie-docs
-    image: mealie-docs
-    ports:
-      - 8888:80
-    build:
-      context: .
-      dockerfile: Dockerfile
-    restart: always

frontend/pages/group/reports/_id.vue

@@ -4,9 +4,7 @@
       <template #header>
         <v-img max-height="200" max-width="200" class="mb-2" :src="require('~/static/svgs/data-reports.svg')"></v-img>
       </template>
-      <template #title> Recipe Data Migrations</template>
+      <template #title> Report </template>
-      Recipes can be migrated from another supported application to Mealie. This is a great way to get started with
-      Mealie.
     </BasePageTitle>
     <v-container v-if="report">
       <BaseCardSectionTitle :title="report.name"> </BaseCardSectionTitle>
@@ -31,8 +29,9 @@
 </template>
 
 <script lang="ts">
-import { defineComponent, useRoute, reactive, toRefs, onMounted } from "@nuxtjs/composition-api";
+import { defineComponent, useRoute, ref, onMounted } from "@nuxtjs/composition-api";
 import { useUserApi } from "~/composables/api";
+import { ReportOut } from "~/types/api-types/reports";
 
 export default defineComponent({
   setup() {
@@ -41,16 +40,11 @@ export default defineComponent({
 
     const api = useUserApi();
 
-    const state = reactive({
+    const report = ref<ReportOut | null>(null);
-      report: {},
-    });
 
     async function getReport() {
       const { data } = await api.groupReports.getOne(id);
-
+      report.value = data ?? null;
-      if (data) {
-        state.report = data;
-      }
     }
 
     onMounted(async () => {
@@ -64,7 +58,7 @@ export default defineComponent({
     ];
 
     return {
-      ...toRefs(state),
+      report,
       id,
       itemHeaders,
     };
@@ -72,5 +66,4 @@ export default defineComponent({
 });
 </script>
 
-<style lang="scss" scoped>
+<style lang="scss" scoped></style>
-</style>

frontend/pages/recipe/create/bulk.vue

@@ -1,15 +1,15 @@
 <template>
   <div>
-    <v-card flat>
+    <div flat>
       <v-card-title class="headline"> Recipe Bulk Importer </v-card-title>
       <v-card-text>
         The Bulk recipe importer allows you to import multiple recipes at once by queing the sites on the backend and
         running the task in the background. This can be useful when initially migrating to Mealie, or when you want to
         import a large number of recipes.
       </v-card-text>
-    </v-card>
+    </div>
     <section class="mt-2">
-      <v-row v-for="(bulkUrl, idx) in bulkUrls" :key="'bulk-url' + idx" class="my-1" dense>
+      <v-row v-for="(_, idx) in bulkUrls" :key="'bulk-url' + idx" class="my-1" dense>
         <v-col cols="12" xs="12" sm="12" md="12">
           <v-text-field
             v-model="bulkUrls[idx].url"
@@ -26,7 +26,7 @@
             class="rounded-lg"
           >
             <template #append>
-              <v-btn color="error" icon x-small @click="bulkUrls.splice(idx, 1)">
+              <v-btn style="margin-top: -2px" icon small @click="bulkUrls.splice(idx, 1)">
                 <v-icon>
                   {{ $globals.icons.delete }}
                 </v-icon>
@@ -34,38 +34,40 @@
             </template>
           </v-text-field>
         </v-col>
-        <v-col cols="12" xs="12" sm="6">
+        <template v-if="showCatTags">
-          <RecipeOrganizerSelector
+          <v-col cols="12" xs="12" sm="6">
-            v-model="bulkUrls[idx].categories"
+            <RecipeOrganizerSelector
-            :items="allCategories || []"
+              v-model="bulkUrls[idx].categories"
-            selector-type="category"
+              :items="allCategories || []"
-            :input-attrs="{
+              selector-type="category"
-              filled: true,
+              :input-attrs="{
-              singleLine: true,
+                filled: true,
-              dense: true,
+                singleLine: true,
-              rounded: true,
+                dense: true,
-              class: 'rounded-lg',
+                rounded: true,
-              hideDetails: true,
+                class: 'rounded-lg',
-              clearable: true,
+                hideDetails: true,
-            }"
+                clearable: true,
-          />
+              }"
-        </v-col>
+            />
-        <v-col cols="12" xs="12" sm="6">
+          </v-col>
-          <RecipeOrganizerSelector
+          <v-col cols="12" xs="12" sm="6">
-            v-model="bulkUrls[idx].tags"
+            <RecipeOrganizerSelector
-            :items="allTags || []"
+              v-model="bulkUrls[idx].tags"
-            selector-type="tag"
+              :items="allTags || []"
-            :input-attrs="{
+              selector-type="tag"
-              filled: true,
+              :input-attrs="{
-              singleLine: true,
+                filled: true,
-              dense: true,
+                singleLine: true,
-              rounded: true,
+                dense: true,
-              class: 'rounded-lg',
+                rounded: true,
-              hideDetails: true,
+                class: 'rounded-lg',
-              clearable: true,
+                hideDetails: true,
-            }"
+                clearable: true,
-          />
+              }"
-        </v-col>
+            />
+          </v-col>
+        </template>
       </v-row>
       <v-card-actions class="justify-end">
         <BaseButton
@@ -78,32 +80,56 @@
           Clear
         </BaseButton>
         <v-spacer></v-spacer>
-        <BaseButton color="info" @click="bulkUrls.push({ url: '', categories: [], tags: [] })">
+        <BaseButton class="mr-1" color="info" @click="bulkUrls.push({ url: '', categories: [], tags: [] })">
-          <template #icon> {{ $globals.icons.createAlt }} </template> New
+          <template #icon> {{ $globals.icons.createAlt }} </template>
+          New
         </BaseButton>
+        <RecipeDialogBulkAdd v-model="bulkDialog" @bulk-data="assignUrls" />
+      </v-card-actions>
+      <div class="px-1">
+        <v-checkbox v-model="showCatTags" hide-details label="Set Categories and Tags " />
+      </div>
+      <v-card-actions class="justify-end">
         <BaseButton :disabled="bulkUrls.length === 0 || lockBulkImport" @click="bulkCreate">
-          <template #icon> {{ $globals.icons.check }} </template> Submit
+          <template #icon> {{ $globals.icons.check }} </template>
+          Submit
         </BaseButton>
       </v-card-actions>
     </section>
+    <section class="mt-12">
+      <BaseCardSectionTitle title="Bulk Imports"> </BaseCardSectionTitle>
+      <ReportTable :items="reports" @delete="deleteReport" />
+    </section>
   </div>
 </template>
 
 <script lang="ts">
 import { defineComponent, reactive, toRefs, ref } from "@nuxtjs/composition-api";
+import { whenever } from "@vueuse/shared";
 import { useUserApi } from "~/composables/api";
 import { alert } from "~/composables/use-toast";
 import RecipeOrganizerSelector from "~/components/Domain/Recipe/RecipeOrganizerSelector.vue";
 import { useCategories, useTags } from "~/composables/recipes";
+import { ReportSummary } from "~/types/api-types/reports";
+import RecipeDialogBulkAdd from "~/components/Domain/Recipe/RecipeDialogBulkAdd.vue";
 
 export default defineComponent({
-  components: { RecipeOrganizerSelector },
+  components: { RecipeOrganizerSelector, RecipeDialogBulkAdd },
   setup() {
     const state = reactive({
       error: false,
       loading: false,
+      showCatTags: false,
+      bulkDialog: false,
     });
 
+    whenever(
+      () => !state.showCatTags,
+      () => {
+        console.log("showCatTags changed");
+      }
+    );
+
     const api = useUserApi();
 
     const bulkUrls = ref([{ url: "", categories: [], tags: [] }]);
@@ -122,6 +148,8 @@ export default defineComponent({
       } else {
         alert.error("Bulk import process has failed");
       }
+
+      fetchReports();
     }
 
     const { allTags, useAsyncGetAll: getAllTags } = useTags();
@@ -130,7 +158,37 @@ export default defineComponent({
     getAllTags();
     getAllCategories();
 
+    // =========================================================
+    // Reports
+
+    const reports = ref<ReportSummary[]>([]);
+
+    async function fetchReports() {
+      const { data } = await api.groupReports.getAll("bulk_import");
+      reports.value = data ?? [];
+    }
+
+    async function deleteReport(id: string) {
+      console.log(id);
+      const { response } = await api.groupReports.deleteOne(id);
+
+      if (response?.status === 200) {
+        fetchReports();
+      } else {
+        alert.error("Report deletion failed");
+      }
+    }
+
+    fetchReports();
+
+    function assignUrls(urls: string[]) {
+      bulkUrls.value = urls.map((url) => ({ url, categories: [], tags: [] }));
+    }
+
     return {
+      assignUrls,
+      reports,
+      deleteReport,
       allTags,
       allCategories,
       bulkCreate,

frontend/pages/recipe/create/debug.vue

@@ -1,7 +1,7 @@
 <template>
   <div>
     <v-form ref="domUrlForm" @submit.prevent="debugUrl(recipeUrl)">
-      <v-card flat>
+      <div>
         <v-card-title class="headline"> Recipe Debugger </v-card-title>
         <v-card-text>
           Grab the URL of the recipe you want to debug and paste it here. The URL will be scraped by the recipe scraper
@@ -32,7 +32,7 @@
             </BaseButton>
           </div>
         </v-card-actions>
-      </v-card>
+      </div>
     </v-form>
     <section v-if="debugData">
       <v-checkbox v-model="debugTreeView" label="Tree View"></v-checkbox>

frontend/pages/recipe/create/new.vue

@@ -1,5 +1,5 @@
 <template>
-  <v-card flat>
+  <div>
     <v-card-title class="headline"> Create Recipe </v-card-title>
     <v-card-text>
       Create a recipe by providing the name. All recipes must have unique names.
@@ -31,7 +31,7 @@
         />
       </div>
     </v-card-actions>
-  </v-card>
+  </div>
 </template>
 <script lang="ts">
 import { defineComponent, reactive, toRefs, ref, useRouter } from "@nuxtjs/composition-api";

frontend/pages/recipe/create/url.vue

@@ -1,7 +1,7 @@
 <template>
   <div>
     <v-form ref="domUrlForm" @submit.prevent="createByUrl(recipeUrl, importKeywordsAsTags)">
-      <v-card flat>
+      <div>
         <v-card-title class="headline"> Scrape Recipe </v-card-title>
         <v-card-text>
           Scrape a recipe by url. Provide the url for the site you want to scrape, and Mealie will attempt to scrape the
@@ -27,7 +27,7 @@
             <BaseButton :disabled="recipeUrl === null" rounded block type="submit" :loading="loading" />
           </div>
         </v-card-actions>
-      </v-card>
+      </div>
     </v-form>
     <v-expand-transition>
       <v-alert v-show="error" color="error" class="mt-6 white--text">

frontend/pages/recipe/create/zip.vue

@@ -1,6 +1,6 @@
 <template>
   <v-form>
-    <v-card flat>
+    <div>
       <v-card-title class="headline"> Import from Zip </v-card-title>
       <v-card-text>
         Import a single recipe that was exported from another Mealie instance.
@@ -25,7 +25,7 @@
           <BaseButton :disabled="newRecipeZip === null" large rounded block :loading="loading" @click="createByZip" />
         </div>
       </v-card-actions>
-    </v-card>
+    </div>
   </v-form>
 </template>
 

frontend/types/api-types/reports.ts

@@ -5,7 +5,7 @@
 /* Do not modify it by hand - just update the pydantic models and then re-run the script
 */
 
-export type ReportCategory = "backup" | "restore" | "migration";
+export type ReportCategory = "backup" | "restore" | "migration" | "bulk_import";
 export type ReportSummaryStatus = "in-progress" | "success" | "failure" | "partial";
 
 export interface ReportCreate {

mealie/routes/groups/controller_group_reports.py

@@ -1,6 +1,6 @@
 from functools import cached_property
 
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
 from pydantic import UUID4
 
 from mealie.core.exceptions import mealie_registered_exceptions
@@ -8,6 +8,7 @@ from mealie.routes._base.base_controllers import BaseUserController
 from mealie.routes._base.controller import controller
 from mealie.routes._base.mixins import HttpRepo
 from mealie.schema.reports.reports import ReportCategory, ReportCreate, ReportOut, ReportSummary
+from mealie.schema.response.responses import ErrorResponse, SuccessResponse
 
 router = APIRouter(prefix="/groups/reports", tags=["Groups: Reports"])
 
@@ -39,6 +40,10 @@ class GroupReportsController(BaseUserController):
     def get_one(self, item_id: UUID4):
         return self.mixins.get_one(item_id)
 
-    @router.delete("/{item_id}", status_code=204)
+    @router.delete("/{item_id}", status_code=200)
     def delete_one(self, item_id: UUID4):
-        self.mixins.delete_one(item_id)  # type: ignore
+        try:
+            self.mixins.delete_one(item_id)  # type: ignore
+            return SuccessResponse.respond("Report deleted.")
+        except Exception as ex:
+            raise HTTPException(500, ErrorResponse.respond("Failed to delete report")) from ex

mealie/routes/recipe/recipe_crud_routes.py

@@ -9,7 +9,6 @@ from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel, Field
 from slugify import slugify
-from sqlalchemy.orm.session import Session
 from starlette.responses import FileResponse
 
 from mealie.core import exceptions
@@ -17,7 +16,6 @@ from mealie.core.dependencies import temporary_zip_path
 from mealie.core.dependencies.dependencies import temporary_dir, validate_recipe_token
 from mealie.core.security import create_recipe_slug_token
 from mealie.pkgs import cache
-from mealie.repos.all_repositories import get_repositories
 from mealie.repos.repository_recipes import RepositoryRecipes
 from mealie.routes._base import BaseUserController, controller
 from mealie.routes._base.mixins import HttpRepo
@@ -29,17 +27,16 @@ from mealie.schema.recipe.recipe_asset import RecipeAsset
 from mealie.schema.recipe.recipe_scraper import ScrapeRecipeTest
 from mealie.schema.recipe.request_helpers import RecipeZipTokenResponse, UpdateImageResponse
 from mealie.schema.response.responses import ErrorResponse
-from mealie.schema.server.tasks import ServerTaskNames
 from mealie.services import urls
 from mealie.services.event_bus_service.event_bus_service import EventBusService
 from mealie.services.event_bus_service.message_types import EventTypes
 from mealie.services.recipe.recipe_data_service import RecipeDataService
 from mealie.services.recipe.recipe_service import RecipeService
 from mealie.services.recipe.template_service import TemplateService
+from mealie.services.scraper.recipe_bulk_scraper import RecipeBulkScraperService
 from mealie.services.scraper.scraped_extras import ScraperContext
 from mealie.services.scraper.scraper import create_from_url
 from mealie.services.scraper.scraper_strategies import RecipeScraperPackage
-from mealie.services.server_tasks.background_executory import BackgroundExecutor
 
 
 class BaseRecipeController(BaseUserController):
@@ -172,39 +169,11 @@ class RecipeController(BaseRecipeController):
     @router.post("/create-url/bulk", status_code=202)
     def parse_recipe_url_bulk(self, bulk: CreateRecipeByUrlBulk, bg_tasks: BackgroundTasks):
         """Takes in a URL and attempts to scrape data and load it into the database"""
-        bg_executor = BackgroundExecutor(self.group.id, self.repos, bg_tasks)
+        bulk_scraper = RecipeBulkScraperService(self.service, self.repos, self.group)
+        report_id = bulk_scraper.get_report_id()
+        bg_tasks.add_task(bulk_scraper.scrape, bulk)
 
-        def bulk_import_func(task_id: int, session: Session) -> None:
+        return {"reportId": report_id}
-            database = get_repositories(session)
-            task = database.server_tasks.get_one(task_id)
-
-            task.append_log("test task has started")
-
-            for b in bulk.imports:
-                try:
-                    recipe, _ = create_from_url(b.url)
-
-                    if b.tags:
-                        recipe.tags = b.tags
-
-                    if b.categories:
-                        recipe.recipe_category = b.categories
-
-                    self.service.create_one(recipe)
-                    task.append_log(f"INFO: Created recipe from url: {b.url}")
-                except Exception as e:
-                    task.append_log(f"Error: Failed to create recipe from url: {b.url}")
-                    task.append_log(f"Error: {e}")
-                    self.deps.logger.error(f"Failed to create recipe from url: {b.url}")
-                    self.deps.logger.error(e)
-                database.server_tasks.update(task.id, task)
-
-            task.set_finished()
-            database.server_tasks.update(task.id, task)
-
-        bg_executor.dispatch(ServerTaskNames.bulk_recipe_import, bulk_import_func)
-
-        return {"details": "task has been started"}
 
     @router.post("/test-scrape-url")
     def test_parse_recipe_url(self, url: ScrapeRecipeTest):

mealie/schema/reports/reports.py

@@ -11,6 +11,7 @@ class ReportCategory(str, enum.Enum):
     backup = "backup"
     restore = "restore"
     migration = "migration"
+    bulk_import = "bulk_import"
 
 
 class ReportSummaryStatus(str, enum.Enum):

mealie/services/scraper/recipe_bulk_scraper.py

@@ -0,0 +1,104 @@
+from pydantic import UUID4
+
+from mealie.repos.repository_factory import AllRepositories
+from mealie.schema.recipe.recipe import CreateRecipeByUrlBulk
+from mealie.schema.reports.reports import ReportCategory, ReportCreate, ReportEntryCreate, ReportSummaryStatus
+from mealie.schema.user.user import GroupInDB
+from mealie.services._base_service import BaseService
+from mealie.services.recipe.recipe_service import RecipeService
+from mealie.services.scraper.scraper import create_from_url
+
+
+class RecipeBulkScraperService(BaseService):
+    report_entries: list[ReportEntryCreate]
+
+    def __init__(self, service: RecipeService, repos: AllRepositories, group: GroupInDB) -> None:
+        self.service = service
+        self.repos = repos
+        self.group = group
+        self.report_entries = []
+
+        super().__init__()
+
+    def get_report_id(self) -> UUID4:
+        import_report = ReportCreate(
+            name="Bulk Import",
+            category=ReportCategory.bulk_import,
+            status=ReportSummaryStatus.in_progress,
+            group_id=self.group.id,
+        )
+
+        self.report = self.repos.group_reports.create(import_report)
+        return self.report.id
+
+    def _add_error_entry(self, message: str, exception: str = "") -> None:
+        self.report_entries.append(
+            ReportEntryCreate(
+                report_id=self.report.id,
+                success=False,
+                message=message,
+                exception=exception,
+            )
+        )
+
+    def _save_all_entries(self) -> None:
+        is_success = True
+        is_failure = True
+
+        for entry in self.report_entries:
+            if is_failure and entry.success:
+                is_failure = False
+
+            if is_success and not entry.success:
+                is_success = False
+
+            self.repos.group_report_entries.create(entry)
+
+        if is_success:
+            self.report.status = ReportSummaryStatus.success
+
+        if is_failure:
+            self.report.status = ReportSummaryStatus.failure
+
+        if not is_success and not is_failure:
+            self.report.status = ReportSummaryStatus.partial
+
+        self.repos.group_reports.update(self.report.id, self.report)
+
+    def scrape(self, urls: CreateRecipeByUrlBulk) -> None:
+        if self.report is None:
+            self.get_report_id()
+
+        for b in urls.imports:
+
+            try:
+                recipe, _ = create_from_url(b.url)
+            except Exception as e:
+                self.service.logger.error(f"failed to scrape url during bulk url import {b.url}")
+                self.service.logger.exception(e)
+                self._add_error_entry(f"failed to scrape url {b.url}", str(e))
+                break
+
+            if b.tags:
+                recipe.tags = b.tags
+
+            if b.categories:
+                recipe.recipe_category = b.categories
+
+            try:
+                self.service.create_one(recipe)
+            except Exception as e:
+                self.service.logger.error(f"Failed to save recipe to database during bulk url import {b.url}")
+                self.service.logger.exception(e)
+                self._add_error_entry("Failed to save recipe to database during bulk url import", str(e))
+
+            self.report_entries.append(
+                ReportEntryCreate(
+                    report_id=self.report.id,
+                    success=True,
+                    message=f"Successfully imported recipe {recipe.name}",
+                    exception="",
+                )
+            )
+
+        self._save_all_entries()