playing with AI import

TandoorRecipes · Jan 20, 2025 · e0b414d · e0b414d
1 parent a56c7c2
commit e0b414d
Show file tree

Hide file tree

Showing 10 changed files with 243 additions and 1,983 deletions.
diff --git a/cookbook/urls.py b/cookbook/urls.py
@@ -111,6 +111,7 @@ def extend(self, r):
     path('api/get_recipe_file/<int:recipe_id>/', api.get_recipe_file, name='api_get_recipe_file'),
     path('api/sync_all/', api.sync_all, name='api_sync'),
     path('api/recipe-from-source/', api.RecipeUrlImportView.as_view(), name='api_recipe_from_source'),
+    path('api/image-to-recipe/', api.ImageToRecipeView.as_view(), name='api_image_to_recipe'),
     path('api/ingredient-from-string/', api.ingredient_from_string, name='api_ingredient_from_string'),
     path('api/share-link/<int:pk>', api.share_link, name='api_share_link'),
     path('api/reset-food-inheritance/', api.reset_food_inheritance, name='api_reset_food_inheritance'),

diff --git a/cookbook/views/api.py b/cookbook/views/api.py
@@ -39,6 +39,7 @@
 from django_scopes import scopes_disabled
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view, OpenApiExample, inline_serializer
+from google import generativeai
 from icalendar import Calendar, Event
 from oauth2_provider.models import AccessToken
 from recipe_scrapers import scrape_html
@@ -1741,7 +1742,7 @@ def post(self, request, *args, **kwargs):
                 if re.match('^(.)*/recipe/[0-9]+/[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', url):
                     recipe_json = requests.get(
                         url.replace('/recipe/', '/api/recipe/').replace(re.split('/recipe/[0-9]+', url)[1],
-                                                                             '') + '?share='
+                                                                        '') + '?share='
                         + re.split('/recipe/[0-9]+', url)[1].replace('/', '')).json()
                     recipe_json = clean_dict(recipe_json, 'id')
                     serialized_recipe = RecipeExportSerializer(data=recipe_json, context={'request': request})
@@ -1811,28 +1812,63 @@ def post(self, request, *args, **kwargs):
 
 
 class ImageToRecipeView(APIView):
-    serializer_class = ImportImageSerializer
-    http_method_names = ['post', 'options']
-    # parser_classes = [MultiPartParser]
+    # serializer_class = ImportImageSerializer
+    # http_method_names = ['post', 'options']
+    parser_classes = [MultiPartParser]
     throttle_classes = [RecipeImportThrottle]
     permission_classes = [CustomIsUser & CustomTokenHasReadWriteScope]
 
+    @extend_schema(request=ImportImageSerializer(many=False), responses=RecipeFromSourceResponseSerializer(many=False))
     def post(self, request, *args, **kwargs):
         """
 
         """
+        print('method called')
         serializer = ImportImageSerializer(data=request.data, partial=True)
         if serializer.is_valid():
-            # generativeai.configure(api_key=GOOGLE_AI_API_KEY)
+            generativeai.configure(api_key=GOOGLE_AI_API_KEY)
+
+            model = generativeai.GenerativeModel('gemini-1.5-flash-latest')
+            img = PIL.Image.open(serializer.validated_data['image'])
+           # response = model.generate_content([
+            #                                      "The Image given is a photograph or screenshot taken of a cooking recipe. The data contained in the image should be converted to a structured json format adhering to the specification given in the schema.org/recipes schema. It is not allowed to make any data up, the response must only contain data given in the original image. The response must be a plain json object without any formatting characters or other unnecessary information. If unsure please return an empty json object {}. Do not wrap the response in a markdown codeblock (like ```json ```)",
+             #                                     img], stream=True)
+            #response.resolve()
+
+            #response_text = response.text
+            #response_text.replace('```json', '')
+            #response_text.replace('```', '')
+
+            response_text = '{"@context": "https://schema.org", "@type": "Recipe", "name": "Pennettine mit Nüssen und Zitrone", "prepTime": "PT25M", "recipeYield": "4", "ingredients": [{"@type": "Quantity", "name": "BARILLA Pennettine", "amount": 500, "unitCode": "G"}, {"@type": "Quantity", "name": "Walnußkerne", "amount": 100, "unitCode": "G"}, {"@type": "Quantity", "name": "Sahne", "amount": "1/2-1", "unitCode": "CUP"}, {"@type": "Quantity", "name": "Butter", "amount": 20, "unitCode": "G"}, {"@type": "Quantity", "name": "Zucker", "amount": 10, "unitCode": "G"}, {"@type": "Quantity", "name": "gemahlener Zimt", "amount": 1, "unitCode": "UNIT"}, {"@type": "Quantity", "name": "Zitrone, unbehandelt", "amount": "1/2-1", "unitCode": "UNIT"}, {"@type": "Quantity", "name": "Salz", "amount": null, "unitCode": null}, {"@type": "Quantity", "name": "frisch gemahlener Pfeffer", "amount": null, "unitCode": null}], "recipeInstructions": [{"@type": "HowToStep", "text": "Walnußkerne hacken, in eine große Schüssel geben, Zimt und Zucker mit einer Prise Salz und Pfeffer untermischen."}, {"@type": "HowToStep", "text": "Die Zitrone waschen, Schale einer 1/2 Zitrone abreiben, dann zusammen mit den anderen Zutaten in die Schüssel geben und zuletzt in Stückchen geschnittene Butter untermischen."}, {"@type": "HowToStep", "text": "Die Mischung schmelzen, indem Sie die Schüssel auf den Topf mit dem kochenden Pasta-Wasser stellen und hin und wieder umrühren."}, {"@type": "HowToStep", "text": "Sahne dazugeben und die Zutaten mit einem Schneebesen verrühren."}, {"@type": "HowToStep", "text": "BARILLA Pennettine in reichlich Salzwasser al dente kochen, abgießen und mit der Sauce anrichten."}]}'
+
+            print(response_text)
+
+            try:
+                data_json = json.loads(response_text)
+                # if '@context' not in data_json:
+                #     data_json['@context'] = 'https://schema.org'
+                # if '@type' not in data_json:
+                #     data_json['@type'] = 'Recipe'
+                data = "<script type='application/ld+json'>" + json.dumps(data_json) + "</script>"
+                #data = "<script type='application/ld+json'>" + response_text + "</script>"
+
+                scrape = scrape_html(html=data, org_url='https://urlnotfound.none', supported_only=False)
+                print(str(scrape.ingredients()))
+                if scrape:
+                    response = {}
+                    response['recipe'] = helper.get_from_scraper(scrape, request)
+                    response['images'] = []
+                    response['duplicates'] = []
+                    return Response(RecipeFromSourceResponseSerializer(context={'request': request}).to_representation(response), status=status.HTTP_200_OK)
+            except JSONDecodeError:
+                traceback.print_exc()
+                print('Jsond dcode error')
+                pass
 
-            # model = generativeai.GenerativeModel('gemini-1.5-flash-latest')
-            # img = PIL.Image.open('')
-            # response = model.generate_content(["The image contains a recipe. Please return all data contained in the recipe formatted according to the schema.org specification for recipes", img], stream=True)
-            # response.resolve()
-            Response({'msg': 'SUCCESS'})
+            # TODO proper serializer response
+            return Response({'msg': 'PARSE_FAIL', 'response': response_text})
         else:
-            Response({'msg': serializer.errors})
-        return Response({'test': 'test'})
+            return Response({'msg': serializer.errors})
 
 
 @extend_schema(

diff --git a/vue3/src/composables/useFileApi.ts b/vue3/src/composables/useFileApi.ts
@@ -1,7 +1,7 @@
 import {useDjangoUrls} from "@/composables/useDjangoUrls";
 import {ref} from "vue";
 import {getCookie} from "@/utils/cookie";
-import {RecipeImageFromJSON, UserFile, UserFileFromJSON} from "@/openapi";
+import {RecipeFromJSON, RecipeFromSourceFromJSON, RecipeFromSourceResponseFromJSON, RecipeImageFromJSON, UserFile, UserFileFromJSON} from "@/openapi";
 import {ErrorMessageType, PreparedMessage, useMessageStore} from "@/stores/MessageStore";
 
 /**
@@ -76,5 +76,28 @@ export function useFileApi() {
         })
     }
 
-    return {fileApiLoading, createOrUpdateUserFile, updateRecipeImage}
+    /**
+     * uploads the given file to the image recognition endpoint
+     * @param file file object to upload
+     */
+    function convertImageToRecipe(file: File){
+        let formData = new FormData()
+        if (file != null) {
+            formData.append('image', file)
+        }
+
+        return fetch(getDjangoUrl(`api/image-to-recipe/`), {
+            method: 'POST',
+            headers: {'X-CSRFToken': getCookie('csrftoken')},
+            body: formData
+        }).then(r => {
+            return r.json().then(r => {
+                return RecipeFromSourceResponseFromJSON(r)
+            })
+        }).finally(() => {
+            fileApiLoading.value = false
+        })
+    }
+
+    return {fileApiLoading, createOrUpdateUserFile, updateRecipeImage, convertImageToRecipe}
 }
diff --git a/vue3/src/openapi/.openapi-generator/FILES b/vue3/src/openapi/.openapi-generator/FILES
@@ -7,8 +7,6 @@ models/AccessToken.ts
 models/AuthToken.ts
 models/AutoMealPlan.ts
 models/Automation.ts
-models/AutomationTypeEnum.ts
-models/BaseUnitEnum.ts
 models/BookmarkletImport.ts
 models/BookmarkletImportList.ts
 models/ConnectorConfigConfig.ts
@@ -22,7 +20,6 @@ models/FoodInheritField.ts
 models/FoodShoppingUpdate.ts
 models/FoodSimple.ts
 models/Group.ts
-models/ImportImage.ts
 models/ImportLog.ts
 models/Ingredient.ts
 models/IngredientString.ts
@@ -34,16 +31,6 @@ models/MealPlan.ts
 models/MealType.ts
 models/MethodEnum.ts
 models/NutritionInformation.ts
-models/OpenDataCategory.ts
-models/OpenDataConversion.ts
-models/OpenDataFood.ts
-models/OpenDataFoodProperty.ts
-models/OpenDataProperty.ts
-models/OpenDataStore.ts
-models/OpenDataStoreCategory.ts
-models/OpenDataUnit.ts
-models/OpenDataUnitTypeEnum.ts
-models/OpenDataVersion.ts
 models/PaginatedAutomationList.ts
 models/PaginatedBookmarkletImportListList.ts
 models/PaginatedCookLogList.ts
@@ -90,13 +77,6 @@ models/PatchedInviteLink.ts
 models/PatchedKeyword.ts
 models/PatchedMealPlan.ts
 models/PatchedMealType.ts
-models/PatchedOpenDataCategory.ts
-models/PatchedOpenDataConversion.ts
-models/PatchedOpenDataFood.ts
-models/PatchedOpenDataProperty.ts
-models/PatchedOpenDataStore.ts
-models/PatchedOpenDataUnit.ts
-models/PatchedOpenDataVersion.ts
 models/PatchedProperty.ts
 models/PatchedPropertyType.ts
 models/PatchedRecipe.ts
@@ -156,6 +136,7 @@ models/SupermarketCategoryRelation.ts
 models/Sync.ts
 models/SyncLog.ts
 models/ThemeEnum.ts
+models/TypeEnum.ts
 models/Unit.ts
 models/UnitConversion.ts
 models/User.ts