feat: Improved Ingredient Matching (#2535)

* added normalization to foods and units

* changed search to reference new normalized fields

* fix tests

* added parsed food matching to backend

* prevent pagination from ordering when searching

* added extra fuzzy matching to sqlite ing matching

* added tests

* only apply search ordering when order_by is null

* enabled post-search fuzzy matching for postgres

* fixed postgres fuzzy search test

* idk why this is failing

* 🤦

* simplified frontend ing matching
and restored automatic unit creation

* tightened food fuzzy threshold

* change to rapidfuzz

* sped up fuzzy matching with process

* fixed units not matching by abbreviation

* fast return for exact matches

* replace db searching with pure fuzz

* added fuzzy normalization

* tightened unit fuzzy matching thresh

* cleaned up comments/var names

* ran matching logic through the dryer

* oops

* simplified order by application logic
This commit is contained in:
Michael Genson 2023-09-15 12:19:34 -05:00 committed by GitHub
parent 084ad4228b
commit 2dfbe9f08d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 738 additions and 97 deletions

View File

@ -0,0 +1,71 @@
"""empty message
Revision ID: 0341b154f79a
Revises: bcfdad6b7355
Create Date: 2023-09-01 14:55:42.166766
"""
import sqlalchemy as sa
from sqlalchemy import orm, select
from alembic import op
from mealie.db.models.recipe.ingredient import IngredientFoodModel, IngredientUnitModel
# revision identifiers, used by Alembic.
revision = "0341b154f79a"
down_revision = "bcfdad6b7355"
branch_labels = None
depends_on = None
def populate_normalized_fields():
bind = op.get_bind()
session = orm.Session(bind=bind)
units = session.execute(select(IngredientUnitModel)).scalars().all()
for unit in units:
if unit.name is not None:
unit.name_normalized = IngredientUnitModel.normalize(unit.name)
if unit.abbreviation is not None:
unit.abbreviation_normalized = IngredientUnitModel.normalize(unit.abbreviation)
session.add(unit)
foods = session.execute(select(IngredientFoodModel)).scalars().all()
for food in foods:
if food.name is not None:
food.name_normalized = IngredientFoodModel.normalize(food.name)
session.add(food)
session.commit()
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("ingredient_foods", sa.Column("name_normalized", sa.String(), nullable=True))
op.create_index(op.f("ix_ingredient_foods_name_normalized"), "ingredient_foods", ["name_normalized"], unique=False)
op.add_column("ingredient_units", sa.Column("name_normalized", sa.String(), nullable=True))
op.add_column("ingredient_units", sa.Column("abbreviation_normalized", sa.String(), nullable=True))
op.create_index(
op.f("ix_ingredient_units_abbreviation_normalized"),
"ingredient_units",
["abbreviation_normalized"],
unique=False,
)
op.create_index(op.f("ix_ingredient_units_name_normalized"), "ingredient_units", ["name_normalized"], unique=False)
# ### end Alembic commands ###
populate_normalized_fields()
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_ingredient_units_name_normalized"), table_name="ingredient_units")
op.drop_index(op.f("ix_ingredient_units_abbreviation_normalized"), table_name="ingredient_units")
op.drop_column("ingredient_units", "abbreviation_normalized")
op.drop_column("ingredient_units", "name_normalized")
op.drop_index(op.f("ix_ingredient_foods_name_normalized"), table_name="ingredient_foods")
op.drop_column("ingredient_foods", "name_normalized")
# ### end Alembic commands ###

View File

@ -205,8 +205,7 @@ export default defineComponent({
async function createAssignFood() { async function createAssignFood() {
foodData.data.name = foodSearch.value; foodData.data.name = foodSearch.value;
await foodStore.actions.createOne(foodData.data); props.value.food = await foodStore.actions.createOne(foodData.data) || undefined;
props.value.food = foodStore.foods.value?.find((food) => food.name === foodSearch.value);
foodData.reset(); foodData.reset();
} }
@ -218,8 +217,7 @@ export default defineComponent({
async function createAssignUnit() { async function createAssignUnit() {
unitsData.data.name = unitSearch.value; unitsData.data.name = unitSearch.value;
await unitStore.actions.createOne(unitsData.data); props.value.unit = await unitStore.actions.createOne(unitsData.data) || undefined;
props.value.unit = unitStore.units.value?.find((unit) => unit.name === unitSearch.value);
unitsData.reset(); unitsData.reset();
} }

View File

@ -13,9 +13,9 @@ interface PublicStoreActions<T extends BoundT> {
} }
interface StoreActions<T extends BoundT> extends PublicStoreActions<T> { interface StoreActions<T extends BoundT> extends PublicStoreActions<T> {
createOne(createData: T): Promise<void>; createOne(createData: T): Promise<T | null>;
updateOne(updateData: T): Promise<void>; updateOne(updateData: T): Promise<T | null>;
deleteOne(id: string | number): Promise<void>; deleteOne(id: string | number): Promise<T | null>;
} }
@ -121,31 +121,34 @@ export function useStoreActions<T extends BoundT>(
if (data && allRef?.value) { if (data && allRef?.value) {
allRef.value.push(data); allRef.value.push(data);
} else { } else {
refresh(); await refresh();
} }
loading.value = false; loading.value = false;
return data;
} }
async function updateOne(updateData: T) { async function updateOne(updateData: T) {
if (!updateData.id) { if (!updateData.id) {
return; return null;
} }
loading.value = true; loading.value = true;
const { data } = await api.updateOne(updateData.id, updateData); const { data } = await api.updateOne(updateData.id, updateData);
if (data && allRef?.value) { if (data && allRef?.value) {
refresh(); await refresh();
} }
loading.value = false; loading.value = false;
return data;
} }
async function deleteOne(id: string | number) { async function deleteOne(id: string | number) {
loading.value = true; loading.value = true;
const { response } = await api.deleteOne(id); const { response } = await api.deleteOne(id);
if (response && allRef?.value) { if (response && allRef?.value) {
refresh(); await refresh();
} }
loading.value = false; loading.value = false;
return response?.data || null;
} }
return { return {

View File

@ -68,7 +68,15 @@
<RecipeIngredientEditor v-model="parsedIng[index].ingredient" allow-insert-ingredient @insert-ingredient="insertIngredient(index)" @delete="deleteIngredient(index)" /> <RecipeIngredientEditor v-model="parsedIng[index].ingredient" allow-insert-ingredient @insert-ingredient="insertIngredient(index)" @delete="deleteIngredient(index)" />
{{ ing.input }} {{ ing.input }}
<v-card-actions> <v-card-actions>
<v-spacer></v-spacer> <v-spacer />
<BaseButton
v-if="errors[index].unitError && errors[index].unitErrorMessage !== ''"
color="warning"
small
@click="createUnit(ing.ingredient.unit, index)"
>
{{ errors[index].unitErrorMessage }}
</BaseButton>
<BaseButton <BaseButton
v-if="errors[index].foodError && errors[index].foodErrorMessage !== ''" v-if="errors[index].foodError && errors[index].foodErrorMessage !== ''"
color="warning" color="warning"
@ -99,7 +107,7 @@ import {
import RecipeIngredientEditor from "~/components/Domain/Recipe/RecipeIngredientEditor.vue"; import RecipeIngredientEditor from "~/components/Domain/Recipe/RecipeIngredientEditor.vue";
import { useUserApi } from "~/composables/api"; import { useUserApi } from "~/composables/api";
import { useRecipe } from "~/composables/recipes"; import { useRecipe } from "~/composables/recipes";
import { useFoodData, useFoodStore, useUnitStore } from "~/composables/store"; import { useFoodData, useFoodStore, useUnitStore, useUnitData } from "~/composables/store";
import { Parser } from "~/lib/api/user/recipes/recipe"; import { Parser } from "~/lib/api/user/recipes/recipe";
import { uuid4 } from "~/composables/use-utils"; import { uuid4 } from "~/composables/use-utils";
@ -215,30 +223,19 @@ export default defineComponent({
const foodStore = useFoodStore(); const foodStore = useFoodStore();
const foodData = useFoodData(); const foodData = useFoodData();
const { units } = useUnitStore(); const unitStore = useUnitStore();
const unitData = useUnitData();
const errors = ref<Error[]>([]); const errors = ref<Error[]>([]);
function checkForUnit(unit?: IngredientUnit | CreateIngredientUnit) { function checkForUnit(unit?: IngredientUnit | CreateIngredientUnit) {
if (!unit) { // @ts-expect-error; we're just checking if there's an id on this unit and returning a boolean
return false; return !!unit?.id;
}
if (units.value && unit?.name) {
const lower = unit.name.toLowerCase();
return units.value.some((u) => u.name.toLowerCase() === lower);
}
return false;
} }
function checkForFood(food?: IngredientFood | CreateIngredientFood) { function checkForFood(food?: IngredientFood | CreateIngredientFood) {
if (!food) { // @ts-expect-error; we're just checking if there's an id on this food and returning a boolean
return false; return !!food?.id;
}
if (foodStore.foods.value && food?.name) {
const lower = food.name.toLowerCase();
return foodStore.foods.value.some((f) => f.name.toLowerCase() === lower);
}
return false;
} }
async function createFood(food: CreateIngredientFood | undefined, index: number) { async function createFood(food: CreateIngredientFood | undefined, index: number) {
@ -247,11 +244,24 @@ export default defineComponent({
} }
foodData.data.name = food.name; foodData.data.name = food.name;
await foodStore.actions.createOne(foodData.data); parsedIng.value[index].ingredient.food = await foodStore.actions.createOne(foodData.data) || undefined;
errors.value[index].foodError = false; errors.value[index].foodError = false;
foodData.reset(); foodData.reset();
} }
async function createUnit(unit: CreateIngredientUnit | undefined, index: number) {
if (!unit) {
return;
}
unitData.data.name = unit.name;
parsedIng.value[index].ingredient.unit = await unitStore.actions.createOne(unitData.data) || undefined;
errors.value[index].unitError = false;
unitData.reset();
}
function insertIngredient(index: number) { function insertIngredient(index: number) {
if (!recipe.value?.recipeIngredient) { if (!recipe.value?.recipeIngredient) {
return; return;
@ -287,27 +297,21 @@ export default defineComponent({
// ========================================================= // =========================================================
// Save All Logic // Save All Logic
async function saveAll() { async function saveAll() {
let ingredients = parsedIng.value.map((ing) => { const ingredients = parsedIng.value.map((ing) => {
if (!checkForFood(ing.ingredient.food)) {
ing.ingredient.food = undefined;
}
if (!checkForUnit(ing.ingredient.unit)) {
ing.ingredient.unit = undefined;
}
return { return {
...ing.ingredient, ...ing.ingredient,
originalText: ing.input, originalText: ing.input,
} as RecipeIngredient; } as RecipeIngredient;
}); });
ingredients = ingredients.map((ing) => {
if (!foodStore.foods.value || !units.value) {
return ing;
}
// Get food from foods
const lowerFood = ing.food?.name?.toLowerCase();
ing.food = foodStore.foods.value.find((f) => f.name.toLowerCase() === lowerFood);
// Get unit from units
const lowerUnit = ing.unit?.name?.toLowerCase();
ing.unit = units.value.find((u) => u.name.toLowerCase() === lowerUnit);
return ing;
});
if (!recipe.value || !recipe.value.slug) { if (!recipe.value || !recipe.value.slug) {
return; return;
} }
@ -328,6 +332,7 @@ export default defineComponent({
parser, parser,
saveAll, saveAll,
createFood, createFood,
createUnit,
deleteIngredient, deleteIngredient,
insertIngredient, insertIngredient,
errors, errors,

View File

@ -2,6 +2,7 @@ from datetime import datetime
from sqlalchemy import DateTime, Integer from sqlalchemy import DateTime, Integer
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from text_unidecode import unidecode
class SqlAlchemyBase(DeclarativeBase): class SqlAlchemyBase(DeclarativeBase):
@ -9,6 +10,10 @@ class SqlAlchemyBase(DeclarativeBase):
created_at: Mapped[datetime | None] = mapped_column(DateTime, default=datetime.now, index=True) created_at: Mapped[datetime | None] = mapped_column(DateTime, default=datetime.now, index=True)
update_at: Mapped[datetime | None] = mapped_column(DateTime, default=datetime.now, onupdate=datetime.now) update_at: Mapped[datetime | None] = mapped_column(DateTime, default=datetime.now, onupdate=datetime.now)
@classmethod
def normalize(cls, val: str) -> str:
return unidecode(val).lower().strip()
class BaseMixins: class BaseMixins:
""" """

View File

@ -4,7 +4,6 @@ import sqlalchemy as sa
from sqlalchemy import Boolean, Float, ForeignKey, Integer, String, event, orm from sqlalchemy import Boolean, Float, ForeignKey, Integer, String, event, orm
from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.orm.session import Session from sqlalchemy.orm.session import Session
from text_unidecode import unidecode
from mealie.db.models._model_base import BaseMixins, SqlAlchemyBase from mealie.db.models._model_base import BaseMixins, SqlAlchemyBase
from mealie.db.models.labels import MultiPurposeLabel from mealie.db.models.labels import MultiPurposeLabel
@ -34,9 +33,56 @@ class IngredientUnitModel(SqlAlchemyBase, BaseMixins):
"RecipeIngredientModel", back_populates="unit" "RecipeIngredientModel", back_populates="unit"
) )
# Automatically updated by sqlalchemy event, do not write to this manually
name_normalized: Mapped[str | None] = mapped_column(sa.String, index=True)
abbreviation_normalized: Mapped[str | None] = mapped_column(String, index=True)
@auto_init() @auto_init()
def __init__(self, **_) -> None: def __init__(self, session: Session, name: str | None = None, abbreviation: str | None = None, **_) -> None:
pass if name is not None:
self.name_normalized = self.normalize(name)
if abbreviation is not None:
self.abbreviation = self.normalize(abbreviation)
tableargs = [
sa.Index(
"ix_ingredient_units_name_normalized",
"name_normalized",
unique=False,
),
sa.Index(
"ix_ingredient_units_abbreviation_normalized",
"abbreviation_normalized",
unique=False,
),
]
if session.get_bind().name == "postgresql":
tableargs.extend(
[
sa.Index(
"ix_ingredient_units_name_normalized_gin",
"name_normalized",
unique=False,
postgresql_using="gin",
postgresql_ops={
"name_normalized": "gin_trgm_ops",
},
),
sa.Index(
"ix_ingredient_units_abbreviation_normalized_gin",
"abbreviation_normalized",
unique=False,
postgresql_using="gin",
postgresql_ops={
"abbreviation_normalized": "gin_trgm_ops",
},
),
]
)
self.__table_args__ = tuple(tableargs)
class IngredientFoodModel(SqlAlchemyBase, BaseMixins): class IngredientFoodModel(SqlAlchemyBase, BaseMixins):
@ -57,10 +103,39 @@ class IngredientFoodModel(SqlAlchemyBase, BaseMixins):
label_id: Mapped[GUID | None] = mapped_column(GUID, ForeignKey("multi_purpose_labels.id"), index=True) label_id: Mapped[GUID | None] = mapped_column(GUID, ForeignKey("multi_purpose_labels.id"), index=True)
label: Mapped[MultiPurposeLabel | None] = orm.relationship(MultiPurposeLabel, uselist=False, back_populates="foods") label: Mapped[MultiPurposeLabel | None] = orm.relationship(MultiPurposeLabel, uselist=False, back_populates="foods")
# Automatically updated by sqlalchemy event, do not write to this manually
name_normalized: Mapped[str | None] = mapped_column(sa.String, index=True)
@api_extras @api_extras
@auto_init() @auto_init()
def __init__(self, **_) -> None: def __init__(self, session: Session, name: str | None = None, **_) -> None:
pass if name is not None:
self.name_normalized = self.normalize(name)
tableargs = [
sa.Index(
"ix_ingredient_foods_name_normalized",
"name_normalized",
unique=False,
),
]
if session.get_bind().name == "postgresql":
tableargs.extend(
[
sa.Index(
"ix_ingredient_foods_name_normalized_gin",
"name_normalized",
unique=False,
postgresql_using="gin",
postgresql_ops={
"name_normalized": "gin_trgm_ops",
},
)
]
)
self.__table_args__ = tuple(tableargs)
class RecipeIngredientModel(SqlAlchemyBase, BaseMixins): class RecipeIngredientModel(SqlAlchemyBase, BaseMixins):
@ -92,10 +167,10 @@ class RecipeIngredientModel(SqlAlchemyBase, BaseMixins):
def __init__(self, session: Session, note: str | None = None, orginal_text: str | None = None, **_) -> None: def __init__(self, session: Session, note: str | None = None, orginal_text: str | None = None, **_) -> None:
# SQLAlchemy events do not seem to register things that are set during auto_init # SQLAlchemy events do not seem to register things that are set during auto_init
if note is not None: if note is not None:
self.note_normalized = unidecode(note).lower().strip() self.note_normalized = self.normalize(note)
if orginal_text is not None: if orginal_text is not None:
self.orginal_text = unidecode(orginal_text).lower().strip() self.orginal_text = self.normalize(orginal_text)
tableargs = [ # base set of indices tableargs = [ # base set of indices
sa.Index( sa.Index(
@ -136,17 +211,41 @@ class RecipeIngredientModel(SqlAlchemyBase, BaseMixins):
self.__table_args__ = tuple(tableargs) self.__table_args__ = tuple(tableargs)
@event.listens_for(RecipeIngredientModel.note, "set") @event.listens_for(IngredientUnitModel.name, "set")
def receive_note(target: RecipeIngredientModel, value: str, oldvalue, initiator): def receive_unit_name(target: IngredientUnitModel, value: str | None, oldvalue, initiator):
if value is not None: if value is not None:
target.note_normalized = unidecode(value).lower().strip() target.name_normalized = IngredientUnitModel.normalize(value)
else:
target.name_normalized = None
@event.listens_for(IngredientUnitModel.abbreviation, "set")
def receive_unit_abbreviation(target: IngredientUnitModel, value: str | None, oldvalue, initiator):
if value is not None:
target.abbreviation_normalized = IngredientUnitModel.normalize(value)
else:
target.abbreviation_normalized = None
@event.listens_for(IngredientFoodModel.name, "set")
def receive_food_name(target: IngredientFoodModel, value: str | None, oldvalue, initiator):
if value is not None:
target.name_normalized = IngredientFoodModel.normalize(value)
else:
target.name_normalized = None
@event.listens_for(RecipeIngredientModel.note, "set")
def receive_ingredient_note(target: RecipeIngredientModel, value: str | None, oldvalue, initiator):
if value is not None:
target.note_normalized = RecipeIngredientModel.normalize(value)
else: else:
target.note_normalized = None target.note_normalized = None
@event.listens_for(RecipeIngredientModel.original_text, "set") @event.listens_for(RecipeIngredientModel.original_text, "set")
def receive_original_text(target: RecipeIngredientModel, value: str, oldvalue, initiator): def receive_ingredient_original_text(target: RecipeIngredientModel, value: str | None, oldvalue, initiator):
if value is not None: if value is not None:
target.original_text_normalized = unidecode(value).lower().strip() target.original_text_normalized = RecipeIngredientModel.normalize(value)
else: else:
target.original_text_normalized = None target.original_text_normalized = None

View File

@ -6,7 +6,6 @@ import sqlalchemy.orm as orm
from sqlalchemy import event from sqlalchemy import event
from sqlalchemy.ext.orderinglist import ordering_list from sqlalchemy.ext.orderinglist import ordering_list
from sqlalchemy.orm import Mapped, mapped_column, validates from sqlalchemy.orm import Mapped, mapped_column, validates
from text_unidecode import unidecode
from mealie.db.models._model_utils.guid import GUID from mealie.db.models._model_utils.guid import GUID
@ -189,10 +188,10 @@ class RecipeModel(SqlAlchemyBase, BaseMixins):
# SQLAlchemy events do not seem to register things that are set during auto_init # SQLAlchemy events do not seem to register things that are set during auto_init
if name is not None: if name is not None:
self.name_normalized = unidecode(name).lower().strip() self.name_normalized = self.normalize(name)
if description is not None: if description is not None:
self.description_normalized = unidecode(description).lower().strip() self.description_normalized = self.normalize(description)
tableargs = [ # base set of indices tableargs = [ # base set of indices
sa.UniqueConstraint("slug", "group_id", name="recipe_slug_group_id_key"), sa.UniqueConstraint("slug", "group_id", name="recipe_slug_group_id_key"),
@ -237,12 +236,12 @@ class RecipeModel(SqlAlchemyBase, BaseMixins):
@event.listens_for(RecipeModel.name, "set") @event.listens_for(RecipeModel.name, "set")
def receive_name(target: RecipeModel, value: str, oldvalue, initiator): def receive_name(target: RecipeModel, value: str, oldvalue, initiator):
target.name_normalized = unidecode(value).lower().strip() target.name_normalized = RecipeModel.normalize(value)
@event.listens_for(RecipeModel.description, "set") @event.listens_for(RecipeModel.description, "set")
def receive_description(target: RecipeModel, value: str, oldvalue, initiator): def receive_description(target: RecipeModel, value: str, oldvalue, initiator):
if value is not None: if value is not None:
target.description_normalized = unidecode(value).lower().strip() target.description_normalized = RecipeModel.normalize(value)
else: else:
target.description_normalized = None target.description_normalized = None

View File

@ -312,6 +312,10 @@ class RepositoryGeneric(Generic[Schema, Model]):
if search: if search:
q = self.add_search_to_query(q, eff_schema, search) q = self.add_search_to_query(q, eff_schema, search)
if not pagination_result.order_by and not search:
# default ordering if not searching
pagination_result.order_by = "created_at"
q, count, total_pages = self.add_pagination_to_query(q, pagination_result) q, count, total_pages = self.add_pagination_to_query(q, pagination_result)
# Apply options late, so they do not get used for counting # Apply options late, so they do not get used for counting
@ -371,16 +375,14 @@ class RepositoryGeneric(Generic[Schema, Model]):
if pagination.page < 1: if pagination.page < 1:
pagination.page = 1 pagination.page = 1
if pagination.order_by: query = self.add_order_by_to_query(query, pagination)
query = self.add_order_by_to_query(query, pagination)
return query.limit(pagination.per_page).offset((pagination.page - 1) * pagination.per_page), count, total_pages return query.limit(pagination.per_page).offset((pagination.page - 1) * pagination.per_page), count, total_pages
def add_order_by_to_query(self, query: Select, pagination: PaginationQuery) -> Select: def add_order_by_to_query(self, query: Select, pagination: PaginationQuery) -> Select:
if not pagination.order_by: if not pagination.order_by:
return query return query
if pagination.order_by == "random": elif pagination.order_by == "random":
# randomize outside of database, since not all db's can set random seeds # randomize outside of database, since not all db's can set random seeds
# this solution is db-independent & stable to paging # this solution is db-independent & stable to paging
temp_query = query.with_only_columns(self.model.id) temp_query = query.with_only_columns(self.model.id)

View File

@ -203,6 +203,10 @@ class RepositoryRecipes(RepositoryGeneric[Recipe, RecipeModel]):
if search: if search:
q = self.add_search_to_query(q, self.schema, search) q = self.add_search_to_query(q, self.schema, search)
if not pagination_result.order_by and not search:
# default ordering if not searching
pagination_result.order_by = "created_at"
q, count, total_pages = self.add_pagination_to_query(q, pagination_result) q, count, total_pages = self.add_pagination_to_query(q, pagination_result)
try: try:

View File

@ -12,10 +12,10 @@ router = APIRouter(prefix="/parser")
class IngredientParserController(BaseUserController): class IngredientParserController(BaseUserController):
@router.post("/ingredients", response_model=list[ParsedIngredient]) @router.post("/ingredients", response_model=list[ParsedIngredient])
def parse_ingredients(self, ingredients: IngredientsRequest): def parse_ingredients(self, ingredients: IngredientsRequest):
parser = get_parser(ingredients.parser) parser = get_parser(ingredients.parser, self.group_id, self.session)
return parser.parse(ingredients.ingredients) return parser.parse(ingredients.ingredients)
@router.post("/ingredient", response_model=ParsedIngredient) @router.post("/ingredient", response_model=ParsedIngredient)
def parse_ingredient(self, ingredient: IngredientRequest): def parse_ingredient(self, ingredient: IngredientRequest):
parser = get_parser(ingredient.parser) parser = get_parser(ingredient.parser, self.group_id, self.session)
return parser.parse([ingredient.ingredient])[0] return parser.parse([ingredient.ingredient])[0]

View File

@ -51,7 +51,8 @@ class IngredientFood(CreateIngredientFood):
created_at: datetime.datetime | None created_at: datetime.datetime | None
update_at: datetime.datetime | None update_at: datetime.datetime | None
_searchable_properties: ClassVar[list[str]] = ["name", "description"] _searchable_properties: ClassVar[list[str]] = ["name_normalized"]
_normalize_search: ClassVar[bool] = True
class Config: class Config:
orm_mode = True orm_mode = True
@ -81,7 +82,8 @@ class IngredientUnit(CreateIngredientUnit):
created_at: datetime.datetime | None created_at: datetime.datetime | None
update_at: datetime.datetime | None update_at: datetime.datetime | None
_searchable_properties: ClassVar[list[str]] = ["name", "abbreviation", "description"] _searchable_properties: ClassVar[list[str]] = ["name_normalized", "abbreviation_normalized"]
_normalize_search: ClassVar[bool] = True
class Config: class Config:
orm_mode = True orm_mode = True

View File

@ -34,7 +34,7 @@ class RecipeSearchQuery(MealieModel):
class PaginationQuery(MealieModel): class PaginationQuery(MealieModel):
page: int = 1 page: int = 1
per_page: int = 50 per_page: int = 50
order_by: str = "created_at" order_by: str | None = None
order_by_null_position: OrderByNullPosition | None = None order_by_null_position: OrderByNullPosition | None = None
order_direction: OrderDirection = OrderDirection.desc order_direction: OrderDirection = OrderDirection.desc
query_filter: str | None = None query_filter: str | None = None

View File

@ -1,20 +1,32 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from fractions import Fraction from fractions import Fraction
from typing import TypeVar
from pydantic import UUID4, BaseModel
from rapidfuzz import fuzz, process
from sqlalchemy.orm import Session
from mealie.core.root_logger import get_logger from mealie.core.root_logger import get_logger
from mealie.db.models.recipe.ingredient import IngredientFoodModel, IngredientUnitModel
from mealie.repos.all_repositories import get_repositories
from mealie.repos.repository_factory import AllRepositories
from mealie.schema.recipe import RecipeIngredient from mealie.schema.recipe import RecipeIngredient
from mealie.schema.recipe.recipe_ingredient import ( from mealie.schema.recipe.recipe_ingredient import (
MAX_INGREDIENT_DENOMINATOR, MAX_INGREDIENT_DENOMINATOR,
CreateIngredientFood, CreateIngredientFood,
CreateIngredientUnit, CreateIngredientUnit,
IngredientConfidence, IngredientConfidence,
IngredientFood,
IngredientUnit,
ParsedIngredient, ParsedIngredient,
RegisteredParser, RegisteredParser,
) )
from mealie.schema.response.pagination import PaginationQuery
from . import brute, crfpp from . import brute, crfpp
logger = get_logger(__name__) logger = get_logger(__name__)
T = TypeVar("T", bound=BaseModel)
class ABCIngredientParser(ABC): class ABCIngredientParser(ABC):
@ -22,6 +34,53 @@ class ABCIngredientParser(ABC):
Abstract class for ingredient parsers. Abstract class for ingredient parsers.
""" """
def __init__(self, group_id: UUID4, session: Session) -> None:
self.group_id = group_id
self.session = session
self._foods_by_name: dict[str, IngredientFood] | None = None
self._units_by_name: dict[str, IngredientUnit] | None = None
@property
def _repos(self) -> AllRepositories:
return get_repositories(self.session)
@property
def foods_by_normalized_name(self) -> dict[str, IngredientFood]:
if self._foods_by_name is None:
foods_repo = self._repos.ingredient_foods.by_group(self.group_id)
query = PaginationQuery(page=1, per_page=-1)
all_foods = foods_repo.page_all(query).items
self._foods_by_name = {IngredientFoodModel.normalize(food.name): food for food in all_foods if food.name}
return self._foods_by_name
@property
def units_by_normalized_name_or_abbreviation(self) -> dict[str, IngredientUnit]:
if self._units_by_name is None:
units_repo = self._repos.ingredient_units.by_group(self.group_id)
query = PaginationQuery(page=1, per_page=-1)
all_units = units_repo.page_all(query).items
self._units_by_name = {
IngredientUnitModel.normalize(unit.name): unit for unit in all_units if unit.name
} | {IngredientUnitModel.normalize(unit.abbreviation): unit for unit in all_units if unit.abbreviation}
return self._units_by_name
@property
def food_fuzzy_match_threshold(self) -> int:
"""Minimum threshold to fuzzy match against a database food search"""
return 85
@property
def unit_fuzzy_match_threshold(self) -> int:
"""Minimum threshold to fuzzy match against a database unit search"""
return 70
@abstractmethod @abstractmethod
def parse_one(self, ingredient_string: str) -> ParsedIngredient: def parse_one(self, ingredient_string: str) -> ParsedIngredient:
... ...
@ -30,19 +89,64 @@ class ABCIngredientParser(ABC):
def parse(self, ingredients: list[str]) -> list[ParsedIngredient]: def parse(self, ingredients: list[str]) -> list[ParsedIngredient]:
... ...
@classmethod
def find_match(cls, match_value: str, *, store_map: dict[str, T], fuzzy_match_threshold: int = 0) -> T | None:
# check for literal matches
if match_value in store_map:
return store_map[match_value]
# fuzzy match against food store
fuzz_result = process.extractOne(match_value, store_map.keys(), scorer=fuzz.ratio)
if fuzz_result is None:
return None
choice, score, _ = fuzz_result
if score < fuzzy_match_threshold:
return None
else:
return store_map[choice]
def find_food_match(self, food: IngredientFood | CreateIngredientFood) -> IngredientFood | None:
if isinstance(food, IngredientFood):
return food
match_value = IngredientFoodModel.normalize(food.name)
return self.find_match(
match_value,
store_map=self.foods_by_normalized_name,
fuzzy_match_threshold=self.food_fuzzy_match_threshold,
)
def find_unit_match(self, unit: IngredientUnit | CreateIngredientUnit) -> IngredientUnit | None:
if isinstance(unit, IngredientUnit):
return unit
match_value = IngredientUnitModel.normalize(unit.name)
return self.find_match(
match_value,
store_map=self.units_by_normalized_name_or_abbreviation,
fuzzy_match_threshold=self.unit_fuzzy_match_threshold,
)
def find_ingredient_match(self, ingredient: ParsedIngredient) -> ParsedIngredient:
if ingredient.ingredient.food and (food_match := self.find_food_match(ingredient.ingredient.food)):
ingredient.ingredient.food = food_match
if ingredient.ingredient.unit and (unit_match := self.find_unit_match(ingredient.ingredient.unit)):
ingredient.ingredient.unit = unit_match
return ingredient
class BruteForceParser(ABCIngredientParser): class BruteForceParser(ABCIngredientParser):
""" """
Brute force ingredient parser. Brute force ingredient parser.
""" """
def __init__(self) -> None:
pass
def parse_one(self, ingredient: str) -> ParsedIngredient: def parse_one(self, ingredient: str) -> ParsedIngredient:
bfi = brute.parse(ingredient) bfi = brute.parse(ingredient)
return ParsedIngredient( parsed_ingredient = ParsedIngredient(
input=ingredient, input=ingredient,
ingredient=RecipeIngredient( ingredient=RecipeIngredient(
unit=CreateIngredientUnit(name=bfi.unit), unit=CreateIngredientUnit(name=bfi.unit),
@ -53,6 +157,8 @@ class BruteForceParser(ABCIngredientParser):
), ),
) )
return self.find_ingredient_match(parsed_ingredient)
def parse(self, ingredients: list[str]) -> list[ParsedIngredient]: def parse(self, ingredients: list[str]) -> list[ParsedIngredient]:
return [self.parse_one(ingredient) for ingredient in ingredients] return [self.parse_one(ingredient) for ingredient in ingredients]
@ -62,9 +168,6 @@ class NLPParser(ABCIngredientParser):
Class for CRFPP ingredient parsers. Class for CRFPP ingredient parsers.
""" """
def __init__(self) -> None:
pass
def _crf_to_ingredient(self, crf_model: crfpp.CRFIngredient) -> ParsedIngredient: def _crf_to_ingredient(self, crf_model: crfpp.CRFIngredient) -> ParsedIngredient:
ingredient = None ingredient = None
@ -87,7 +190,7 @@ class NLPParser(ABCIngredientParser):
note=crf_model.input, note=crf_model.input,
) )
return ParsedIngredient( parsed_ingredient = ParsedIngredient(
input=crf_model.input, input=crf_model.input,
ingredient=ingredient, ingredient=ingredient,
confidence=IngredientConfidence( confidence=IngredientConfidence(
@ -97,6 +200,8 @@ class NLPParser(ABCIngredientParser):
), ),
) )
return self.find_ingredient_match(parsed_ingredient)
def parse(self, ingredients: list[str]) -> list[ParsedIngredient]: def parse(self, ingredients: list[str]) -> list[ParsedIngredient]:
crf_models = crfpp.convert_list_to_crf_model(ingredients) crf_models = crfpp.convert_list_to_crf_model(ingredients)
return [self._crf_to_ingredient(crf_model) for crf_model in crf_models] return [self._crf_to_ingredient(crf_model) for crf_model in crf_models]
@ -112,9 +217,9 @@ __registrar = {
} }
def get_parser(parser: RegisteredParser) -> ABCIngredientParser: def get_parser(parser: RegisteredParser, group_id: UUID4, session: Session) -> ABCIngredientParser:
""" """
get_parser returns an ingrdeint parser based on the string enum value get_parser returns an ingrdeint parser based on the string enum value
passed in. passed in.
""" """
return __registrar.get(parser, NLPParser)() return __registrar.get(parser, NLPParser)(group_id, session)

120
poetry.lock generated
View File

@ -605,6 +605,7 @@ files = [
{file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"},
{file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"},
{file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"},
{file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"},
{file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"},
{file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"},
{file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"},
@ -613,6 +614,7 @@ files = [
{file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"},
{file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"},
{file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"},
{file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"},
{file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"},
{file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"},
{file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"},
@ -642,6 +644,7 @@ files = [
{file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"},
{file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"},
{file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"},
{file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"},
{file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"},
{file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"},
{file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"},
@ -650,6 +653,7 @@ files = [
{file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"},
{file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"},
{file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"},
{file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"},
{file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"},
{file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"},
{file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"},
@ -2036,6 +2040,7 @@ files = [
{file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
{file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
{file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
{file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
{file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
{file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
{file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
@ -2043,8 +2048,15 @@ files = [
{file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
{file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
{file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
{file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
{file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
{file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
{file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
{file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
{file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
@ -2061,6 +2073,7 @@ files = [
{file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
{file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
{file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
{file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
{file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
{file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
{file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
@ -2068,6 +2081,7 @@ files = [
{file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
{file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
{file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
{file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
{file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
{file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
{file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
@ -2087,6 +2101,110 @@ files = [
[package.dependencies] [package.dependencies]
pyyaml = "*" pyyaml = "*"
[[package]]
name = "rapidfuzz"
version = "3.2.0"
description = "rapid fuzzy string matching"
optional = false
python-versions = ">=3.7"
files = [
{file = "rapidfuzz-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f5787f1cc456207dee1902804209e1a90df67e88517213aeeb1b248822413b4c"},
{file = "rapidfuzz-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e8d91137b0b5a6ef06c3979b6302265129dee1741486b6baa241ac63a632bea7"},
{file = "rapidfuzz-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c130e73e0079f403b7c3dbf6f85816a3773971c3e639f7289f8b4337b8fd70fe"},
{file = "rapidfuzz-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e18059188bfe3cdbc3462aeec2fa3302b08717e04ca34e2cc6e02fb3c0280d8"},
{file = "rapidfuzz-3.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:37bb6bd6a79d5524f121ff2a7d7df4491519b3f43565dccd4596bd75aa73ab7c"},
{file = "rapidfuzz-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ca0d6aee42effaf2e8883d2181196dd0957b1af5731b0763f10f994c32c823db"},
{file = "rapidfuzz-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:49fc2cbbf05bfa1af3fe4c0e0c8e5c8ac118d6b6ddfb0081cff48ad53734f7ac"},
{file = "rapidfuzz-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bd4fdee46f6ba7d254dba8e7e8f33012c964fc891a06b036b0fd20cab0db301"},
{file = "rapidfuzz-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ab2863732eafd1cc58f249f145c20ad13d4c902d3ef3a369b00438c05e5bfb55"},
{file = "rapidfuzz-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a9658c545de62ac948027092ba7f4e8507ebc5c9aef964eca654409c58f207f0"},
{file = "rapidfuzz-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5f3e36cfadaf29f081ad4ca476e320b639d610e930e0557f395780c9b2bdb135"},
{file = "rapidfuzz-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:239ffc04328e14f5e4097102bd934352a43d5912acf34fb7d3e3fe306de92787"},
{file = "rapidfuzz-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b56ce39ba0a77501d491bc20a2266989ae0264452758b004950ee5f4c10c641f"},
{file = "rapidfuzz-3.2.0-cp310-cp310-win32.whl", hash = "sha256:dbebd639579ab113644699fe0c536ae00aba15b224e40a79987684333d1104a5"},
{file = "rapidfuzz-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:88e99229c4df99a7e5810d4d361033b44e29d8eb4faaddcfb8e4bdcb604cf40a"},
{file = "rapidfuzz-3.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:8e39c4e2e85828aa6c39cc7f30e2917d991b40190a2a3af1fa02396a3362a54e"},
{file = "rapidfuzz-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2f2e618389427c5e8304357a78f83df22558e61f11bc21aeb95dd544c274d330"},
{file = "rapidfuzz-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a2a6babfe4d3ce2eadd0079ee7861cb5f1584845c5a3394edead85457e7d7464"},
{file = "rapidfuzz-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f223deb06895c9c136b40cd8fd7e96ee745c3bb9ed502d7367f6ad9ab6fdd40e"},
{file = "rapidfuzz-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0de6962b45f761355fa4b37de635e4df467d57530732a40d82e748a5bc911731"},
{file = "rapidfuzz-3.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76953516cb3b75fb1234c5a90e0b86be4525f055a9e276237adb1ffe40dca536"},
{file = "rapidfuzz-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1e04861dddbb477500449dc67fb037656a049b6f78c4c434c6000e64aa42bb4"},
{file = "rapidfuzz-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ff6e725eec9c769f9d22126c80a6ada90275c0d693eca2b35d5933178bda5a2"},
{file = "rapidfuzz-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f21ce33242e579ba255c8a8b438782164acaa55bf188d9410298c40cbaa07d5"},
{file = "rapidfuzz-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:986a7aad18768b920bb710e15ed7629d1da0af31589348c0a51d152820efc05d"},
{file = "rapidfuzz-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6e98f0a6fac14b7b9893147deceae12131f6ff169ae1c973635ef97617949c8f"},
{file = "rapidfuzz-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5dd5c4b9f5cd8a8271a90d1bab643028e7172808c68ed5d8dde661a3e51098e3"},
{file = "rapidfuzz-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e336b0a81c5a8e689edf6928136d19e791733a66509026d9acbaa148238186e0"},
{file = "rapidfuzz-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8fa44afb731535a803c4c15ee846257fef050768af96d1d6c0eadb30285d0f7b"},
{file = "rapidfuzz-3.2.0-cp311-cp311-win32.whl", hash = "sha256:d04ad155dbecc0c143912f691d38d4790e290c2ce5411b146c0e00d4f4afd26f"},
{file = "rapidfuzz-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:b9e79e27344af95a71a3bb6cd3562581da5d0780ff847a13ad69ee622d940d3c"},
{file = "rapidfuzz-3.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:dc53747e73f34e8f3a3c1b0bc5b437b90a2c69d873e97781aa7c06543201409a"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:613c1043332eeba0c0910de71af221ac10d820b4fa9615b0083c733b90a757f9"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0907f87beca70e44f78e318eede2416ddba19ec43d28af9248617e8a1741ef3"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcfd184e0b5c58497cc3d961f49ac07ae1656d161c6c4d06230d267ae4e11f00"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7a7d53a2f1ccfb169be26fa3824b1b185420592c75853f16c6b7115315ea6784"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2eac585803c4e8132ed5f4a150621db05c418304982c88cf706abdded65e1632"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc859f654b350def5df2ebc6d09f822b04399823e3dad1c3f2e8776c825fcde7"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8a165f64c528edc0bbbd09c76d64efd4dbe4240fd1961710b69586ef40486e79"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:56a392b655597ecf40535b56bfb7c0856c10c0abc0cbc369fd25a1665420710b"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:5863b176da42b1bb450a28375ef1502f81fbecd210a5aae295d7f2221284ad41"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:8f8590c39a3f745b314f2697b140c8f8600fe7ecfb2101e9e4ec6e7716c66827"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:da00990adf1fbc0904f22409b3451473fa465a0ef49f3075703c206080aa31b2"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:2504205552bf568ac478f17dd612d0e31c4a82c645c66209a442df7e572b5adc"},
{file = "rapidfuzz-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:af3ac648232c109e36c8b941106d726969972644aa3ef55218c5988aa1daea03"},
{file = "rapidfuzz-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:04d22f6058ce5d620ec4ecd771e44cfa77d571137d6c6547df57bdfc44ee2a98"},
{file = "rapidfuzz-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac7ddcd372ed202d1b59b117506da695b291f135435cfbf3e71490aa8e687173"},
{file = "rapidfuzz-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fd3fca0224b84350f73eab1fb5728c58fd25ee4f20e512607c7d83f9bc836d3f"},
{file = "rapidfuzz-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bdb1f92c4666c7e1d3c21268b931cf3f06f32af98dfdeb37641159b15fa31dd"},
{file = "rapidfuzz-3.2.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:871052405c465a45b53a3dc854a8be62079f42cdbb052651ff0b65e2452131e6"},
{file = "rapidfuzz-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb9bb1af5680741cf974f510fb3894907a1b308e819aff3d9ea10b5326e8a5f6"},
{file = "rapidfuzz-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84ce2e010677835fa5ba591419e4404f11a1446f33eec3724a2bff557ae5144a"},
{file = "rapidfuzz-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c13107e0fdca5ccae70659f45646d57453338a9dfc6b152fb7372e4bf73466a0"},
{file = "rapidfuzz-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:538027685a1a8f1699e329f6443951267f169bfa149298734ea679db8f0e7171"},
{file = "rapidfuzz-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3557736672115d082979a8a12f884ed5b24268f4471fee85cfb2ec7212b68607"},
{file = "rapidfuzz-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6bc5e3da74644cf75663f5b438e0ae79b67d1f96d082cda771b0ecfed0528f40"},
{file = "rapidfuzz-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:d2d0fc98d9d7bba44f929d201c2c2c35eb69ea2ffef43d939b297dafef934625"},
{file = "rapidfuzz-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2bf85a3bf34f27383691e8af0fd148b2a3a89f1444d4640d04ef58030f596ee0"},
{file = "rapidfuzz-3.2.0-cp38-cp38-win32.whl", hash = "sha256:cf5ea3f1d65a0bee707245a0096c3a6f769b3ad6f1b9afc7176dfb73eb0ac98f"},
{file = "rapidfuzz-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:54906095444ea8b0a4013f3799b3f2c380205d7f60b9c55774e7d2264fa8d9c6"},
{file = "rapidfuzz-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6d44218823533e0d47770feef86c73c90a6f7e8d4923eafabf56a1fa3444eda0"},
{file = "rapidfuzz-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:87c3d4077e61c66d5dd11198a317f83db8e8cf034239baa16e4384037b611652"},
{file = "rapidfuzz-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc0e1142350566349c41173685988d942ebc89578f25ee27750d261e7d79e1ce"},
{file = "rapidfuzz-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de44a378751fdfb19ddf6af412b3395db4b21ab61f40139f815c82f1a1611b50"},
{file = "rapidfuzz-3.2.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca0983b30c7b289f540b11cdb550e301b3f2e8f0ef9df866aa24a16f6cd96041"},
{file = "rapidfuzz-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:adfffb79288437006be412d74e28cddd7c5e6cc9f84a34aa9c356b13dc1ad2c9"},
{file = "rapidfuzz-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a284386652efb3b7d41ed5dd101ab4ce5936f585c52a47fa9838fc0342235700"},
{file = "rapidfuzz-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c546c83d6bc9006b86f56921b92c3e16d8ddeb4e1663653e755a5d8a3ac258da"},
{file = "rapidfuzz-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:53b3575fa398a5021192c1592dce98965560ad00690be3ade056eab99288562c"},
{file = "rapidfuzz-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:366ade5d0067dc6281e2a6c9e5c91bbfe023b09cef86894de8fe480b4696e3bf"},
{file = "rapidfuzz-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f946dec03cc2c77bc091d186c007d1e957d1f16a4d68a181f5fa75aea40bdf87"},
{file = "rapidfuzz-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:045e5cccb0e792005d5465de0ea4621b9b67778580e558f266984704e68b0087"},
{file = "rapidfuzz-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fd80288b9538c87209893f0934563c20b6a43acf30693794bcc111b294447ee9"},
{file = "rapidfuzz-3.2.0-cp39-cp39-win32.whl", hash = "sha256:a359436754ed5dd10d88706f076caa7f8e5c1469bf5ebba1897dc87aa9ff953e"},
{file = "rapidfuzz-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:75df3d9b895910ee810b2c96c8626cc2b5b63bb237762db36ff79fb466eccc43"},
{file = "rapidfuzz-3.2.0-cp39-cp39-win_arm64.whl", hash = "sha256:893833a903875a50acdbcb7ed33b5426ba47412bd18b3eb80d56d982b641dc59"},
{file = "rapidfuzz-3.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3002c3660180747243cccb40c95ade1960e6665b340f211a114f5994b345ab53"},
{file = "rapidfuzz-3.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa50de7e0f95e1400b2bf38cfeb6e40cf87c862537871c2f7b2050b5db0a9dfc"},
{file = "rapidfuzz-3.2.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54842a578a2a8e5258812a9032ffb55e6f1185490fd160cae64e57b4dc342297"},
{file = "rapidfuzz-3.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:108861623838cd574b0faa3309ce8525c2086159de7f9e23ac263a987c070ebd"},
{file = "rapidfuzz-3.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:d39128415f0b52be08c15eeee5f79288189933a4d6fa5dc5fff11e20614b7989"},
{file = "rapidfuzz-3.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3af2b75635f33ffab84e295773c84a176d4cba75311d836ad79b6795e9da11ac"},
{file = "rapidfuzz-3.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68c678f7f3ca3d83d1e1dd7fb7db3232037d9eef12a47f1d5fe248a76ca47571"},
{file = "rapidfuzz-3.2.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25d2bd257034e910df0951cdeff337dbd086d7d90af3ed9f6721e7bba9fc388a"},
{file = "rapidfuzz-3.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c7f20e68cad26fc140c6f2ac9e8f2632a0cd66e407ba3ea4ace63c669fd4719"},
{file = "rapidfuzz-3.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:f09fd9dc73180deb9ca1c4fbd9cc27378f0ab6ee74e97318c38c5080708702b6"},
{file = "rapidfuzz-3.2.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:af7914fc7683f921492f32314cfbe915a5376cc08a982e09084cbd9b866c9fd4"},
{file = "rapidfuzz-3.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08a242c4b909abbcfa44504dc5041d5eeca4cd088ae51afd6a52b4dc61684fa2"},
{file = "rapidfuzz-3.2.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:71b07afaca28398b93d727a2565491c455896898b66daee4664acde4af94e557"},
{file = "rapidfuzz-3.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24e4c4a031c50e4eeb4787263319a0ac5bed20f4a263d28eac060150e3ba0018"},
{file = "rapidfuzz-3.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d19c2853a464c7b98cc408654412fd875b030f78023ccbefc4ba9eec754e07e7"},
{file = "rapidfuzz-3.2.0.tar.gz", hash = "sha256:448d031d9960fea7826d42bd4284156fc68d3b55a6946eb34ca5c6acf960577b"},
]
[package.extras]
full = ["numpy"]
[[package]] [[package]]
name = "rdflib" name = "rdflib"
version = "6.2.0" version = "6.2.0"
@ -2933,4 +3051,4 @@ pgsql = ["psycopg2-binary"]
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.0"
python-versions = "^3.10" python-versions = "^3.10"
content-hash = "b18a48b2cf3cf26a5eea456056c9dcaf527fb9a6aabf493f25fedb73917175c5" content-hash = "e338e2d0ab5605c2096788cca6e344eead299963335f4ed9f919b2e3f3cb6d3f"

View File

@ -44,6 +44,7 @@ uvicorn = {extras = ["standard"], version = "^0.21.0"}
beautifulsoup4 = "^4.11.2" beautifulsoup4 = "^4.11.2"
isodate = "^0.6.1" isodate = "^0.6.1"
text-unidecode = "^1.3" text-unidecode = "^1.3"
rapidfuzz = "^3.2.0"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
black = "^23.7.0" black = "^23.7.0"

View File

@ -25,13 +25,11 @@ def search_units(database: AllRepositories, unique_local_group_id: str) -> list[
SaveIngredientUnit( SaveIngredientUnit(
group_id=unique_local_group_id, group_id=unique_local_group_id,
name="Table Spoon", name="Table Spoon",
description="unique description",
abbreviation="tbsp", abbreviation="tbsp",
), ),
SaveIngredientUnit( SaveIngredientUnit(
group_id=unique_local_group_id, group_id=unique_local_group_id,
name="Cup", name="Cup",
description="A bucket that's full",
), ),
SaveIngredientUnit( SaveIngredientUnit(
group_id=unique_local_group_id, group_id=unique_local_group_id,
@ -45,6 +43,10 @@ def search_units(database: AllRepositories, unique_local_group_id: str) -> list[
group_id=unique_local_group_id, group_id=unique_local_group_id,
name="Unit with a pretty cool name", name="Unit with a pretty cool name",
), ),
SaveIngredientUnit(
group_id=unique_local_group_id,
name="Unit with a correct horse battery staple",
),
] ]
# Add a bunch of units for stable randomization # Add a bunch of units for stable randomization
@ -64,16 +66,14 @@ def search_units(database: AllRepositories, unique_local_group_id: str) -> list[
(random_string(), []), (random_string(), []),
("Cup", ["Cup"]), ("Cup", ["Cup"]),
("tbsp", ["Table Spoon"]), ("tbsp", ["Table Spoon"]),
("unique description", ["Table Spoon"]),
("very cool name", ["Unit with a very cool name", "Unit with a pretty cool name"]), ("very cool name", ["Unit with a very cool name", "Unit with a pretty cool name"]),
('"Tea Spoon"', ["Tea Spoon"]), ('"Tea Spoon"', ["Tea Spoon"]),
("full bucket", ["Cup"]), ("correct staple", ["Unit with a correct horse battery staple"]),
], ],
ids=[ ids=[
"no_match", "no_match",
"search_by_name", "search_by_name",
"search_by_unit", "search_by_unit",
"search_by_description",
"match_order", "match_order",
"literal_search", "literal_search",
"token_separation", "token_separation",
@ -110,7 +110,7 @@ def test_fuzzy_search(
repo = database.ingredient_units.by_group(unique_local_group_id) repo = database.ingredient_units.by_group(unique_local_group_id)
pagination = PaginationQuery(page=1, per_page=-1, order_by="created_at", order_direction=OrderDirection.asc) pagination = PaginationQuery(page=1, per_page=-1, order_by="created_at", order_direction=OrderDirection.asc)
results = repo.page_all(pagination, search="unique decsription").items results = repo.page_all(pagination, search="tabel spoone").items
assert results and results[0].name == "Table Spoon" assert results and results[0].name == "Table Spoon"

View File

@ -3,9 +3,25 @@ from dataclasses import dataclass
from fractions import Fraction from fractions import Fraction
import pytest import pytest
from pydantic import UUID4
from mealie.db.db_setup import session_context
from mealie.repos.repository_factory import AllRepositories
from mealie.schema.recipe.recipe_ingredient import (
CreateIngredientFood,
CreateIngredientUnit,
IngredientFood,
IngredientUnit,
ParsedIngredient,
RecipeIngredient,
SaveIngredientFood,
SaveIngredientUnit,
)
from mealie.schema.user.user import GroupBase
from mealie.services.parser_services import RegisteredParser, get_parser from mealie.services.parser_services import RegisteredParser, get_parser
from mealie.services.parser_services.crfpp.processor import CRFIngredient, convert_list_to_crf_model from mealie.services.parser_services.crfpp.processor import CRFIngredient, convert_list_to_crf_model
from tests.utils.factories import random_int, random_string
from tests.utils.fixture_schemas import TestUser
@dataclass @dataclass
@ -21,6 +37,70 @@ def crf_exists() -> bool:
return shutil.which("crf_test") is not None return shutil.which("crf_test") is not None
def build_parsed_ing(food: str | None, unit: str | None) -> ParsedIngredient:
ing = RecipeIngredient(unit=None, food=None)
if food:
ing.food = CreateIngredientFood(name=food)
if unit:
ing.unit = CreateIngredientUnit(name=unit)
return ParsedIngredient(input=None, ingredient=ing)
@pytest.fixture()
def unique_local_group_id(database: AllRepositories) -> UUID4:
return str(database.groups.create(GroupBase(name=random_string())).id)
@pytest.fixture()
def parsed_ingredient_data(
database: AllRepositories, unique_local_group_id: UUID4
) -> tuple[list[IngredientFood], list[IngredientUnit]]:
foods = database.ingredient_foods.create_many(
[
SaveIngredientFood(name="potatoes", group_id=unique_local_group_id),
SaveIngredientFood(name="onion", group_id=unique_local_group_id),
SaveIngredientFood(name="green onion", group_id=unique_local_group_id),
SaveIngredientFood(name="frozen pearl onions", group_id=unique_local_group_id),
SaveIngredientFood(name="bell peppers", group_id=unique_local_group_id),
SaveIngredientFood(name="red pepper flakes", group_id=unique_local_group_id),
SaveIngredientFood(name="fresh ginger", group_id=unique_local_group_id),
SaveIngredientFood(name="ground ginger", group_id=unique_local_group_id),
SaveIngredientFood(name="ñör̃m̈ãl̈ĩz̈ẽm̈ẽ", group_id=unique_local_group_id),
]
)
foods.extend(
database.ingredient_foods.create_many(
[
SaveIngredientFood(name=f"{random_string()} food", group_id=unique_local_group_id)
for _ in range(random_int(10, 15))
]
)
)
units = database.ingredient_units.create_many(
[
SaveIngredientUnit(name="Cups", group_id=unique_local_group_id),
SaveIngredientUnit(name="Tablespoon", group_id=unique_local_group_id),
SaveIngredientUnit(name="Teaspoon", group_id=unique_local_group_id),
SaveIngredientUnit(name="Stalk", group_id=unique_local_group_id),
SaveIngredientUnit(name="My Very Long Unit Name", abbreviation="mvlun", group_id=unique_local_group_id),
]
)
units.extend(
database.ingredient_foods.create_many(
[
SaveIngredientUnit(name=f"{random_string()} unit", group_id=unique_local_group_id)
for _ in range(random_int(10, 15))
]
)
)
return foods, units
# TODO - add more robust test cases # TODO - add more robust test cases
test_ingredients = [ test_ingredients = [
TestIngredient("½ cup all-purpose flour", 0.5, "cup", "all-purpose flour", ""), TestIngredient("½ cup all-purpose flour", 0.5, "cup", "all-purpose flour", ""),
@ -47,7 +127,7 @@ def test_nlp_parser():
assert model.unit == test_ingredient.unit assert model.unit == test_ingredient.unit
def test_brute_parser(): def test_brute_parser(unique_user: TestUser):
# input: (quantity, unit, food, comments) # input: (quantity, unit, food, comments)
expectations = { expectations = {
# Dutch # Dutch
@ -67,12 +147,161 @@ def test_brute_parser():
"fresh or frozen", "fresh or frozen",
), ),
} }
parser = get_parser(RegisteredParser.brute)
for key, val in expectations.items(): with session_context() as session:
parsed = parser.parse_one(key) parser = get_parser(RegisteredParser.brute, unique_user.group_id, session)
assert parsed.ingredient.quantity == val[0] for key, val in expectations.items():
assert parsed.ingredient.unit.name == val[1] parsed = parser.parse_one(key)
assert parsed.ingredient.food.name == val[2]
assert parsed.ingredient.note in {val[3], None} assert parsed.ingredient.quantity == val[0]
assert parsed.ingredient.unit.name == val[1]
assert parsed.ingredient.food.name == val[2]
assert parsed.ingredient.note in {val[3], None}
@pytest.mark.parametrize(
"input, expected_unit_name, expected_food_name, expect_unit_match, expect_food_match",
(
pytest.param(
build_parsed_ing(unit="cup", food="potatoes"),
"Cups",
"potatoes",
True,
True,
id="basic match",
),
pytest.param( # this should work in sqlite since "potato" is contained within "potatoes"
build_parsed_ing(unit="cup", food="potato"),
"Cups",
"potatoes",
True,
True,
id="basic fuzzy match",
),
pytest.param(
build_parsed_ing(unit="tablespoon", food="onion"),
"Tablespoon",
"onion",
True,
True,
id="nested match 1",
),
pytest.param(
build_parsed_ing(unit="teaspoon", food="green onion"),
"Teaspoon",
"green onion",
True,
True,
id="nested match 2",
),
pytest.param(
build_parsed_ing(unit="cup", food="gren onion"),
"Cups",
"green onion",
True,
True,
id="nested match 3",
),
pytest.param(
build_parsed_ing(unit="stalk", food="very unique"),
"Stalk",
"very unique",
True,
False,
id="no food match",
),
pytest.param(
build_parsed_ing(unit="cup", food=None),
"Cups",
None,
True,
False,
id="no food input",
),
pytest.param(
build_parsed_ing(unit="very unique", food="fresh ginger"),
"very unique",
"fresh ginger",
False,
True,
id="no unit match",
),
pytest.param(
build_parsed_ing(unit=None, food="potatoes"),
None,
"potatoes",
False,
True,
id="no unit input",
),
pytest.param(
build_parsed_ing(unit="very unique", food="very unique"),
"very unique",
"very unique",
False,
False,
id="no matches",
),
pytest.param(
build_parsed_ing(unit=None, food=None),
None,
None,
False,
False,
id="no input",
),
pytest.param(
build_parsed_ing(unit="mvlun", food="potatoes"),
"My Very Long Unit Name",
"potatoes",
True,
True,
id="unit abbreviation",
),
pytest.param(
build_parsed_ing(unit=None, food="n̅ōr̅m̄a̅l̄i̅z̄e̅m̄e̅"),
None,
"ñör̃m̈ãl̈ĩz̈ẽm̈ẽ",
False,
True,
id="normalization",
),
),
)
def test_parser_ingredient_match(
expected_food_name: str | None,
expected_unit_name: str | None,
expect_food_match: bool,
expect_unit_match: bool,
input: ParsedIngredient,
parsed_ingredient_data: tuple[list[IngredientFood], list[IngredientUnit]], # required so database is populated
unique_local_group_id: UUID4,
):
with session_context() as session:
parser = get_parser(RegisteredParser.brute, unique_local_group_id, session)
parsed_ingredient = parser.find_ingredient_match(input)
if expected_food_name:
assert parsed_ingredient.ingredient.food and parsed_ingredient.ingredient.food.name == expected_food_name
else:
assert parsed_ingredient.ingredient.food is None
if expect_food_match:
assert isinstance(parsed_ingredient.ingredient.food, IngredientFood)
else:
assert parsed_ingredient.ingredient.food is None or isinstance(
parsed_ingredient.ingredient.food, CreateIngredientFood
)
if expected_unit_name:
assert parsed_ingredient.ingredient.unit and parsed_ingredient.ingredient.unit.name == expected_unit_name
else:
assert parsed_ingredient.ingredient.unit is None
if expect_unit_match:
assert isinstance(parsed_ingredient.ingredient.unit, IngredientUnit)
else:
assert parsed_ingredient.ingredient.unit is None or isinstance(
parsed_ingredient.ingredient.unit, CreateIngredientUnit
)