stephen.yu
/
RecipeCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

second commit

master
stephen.yu 9 months ago
parent
5063429ab8
commit
50abf465c8
2 changed files with 219 additions and 223 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 76
      Recipe/models.py
  2. 366
      Recipe/views.py

76
Recipe/models.py
Unescape View File

@ -5,41 +5,41 @@ from django.contrib.postgres.fields import JSONField
# Create your models here. # Create your models here.
class Dish(models.Model): class Dish(models.Model):
name = models.CharField(max_length=255, unique=True) # 假设每个菜名是唯一的 name = models.CharField(max_length=255, unique=True, help_text='菜名')
image = models.TextField() # 存储图片的 Base64 编码 image = models.TextField(help_text='菜品圖片')
likes = models.IntegerField(default=0) likes = models.IntegerField(default=0, help_text='按讚數')
tags = models.TextField() # 可以存储标签列表，例如以逗号分隔的字符串 tags = models.TextField(help_text='標籤')
indications = models.TextField() indications = models.TextField(help_text='適應症')
Calories = models.CharField(max_length=255, unique=False, blank=True, null=True) Calories = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='熱量')
Salt = models.CharField(max_length=255, unique=False, blank=True, null=True) Salt = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='食鹽相當量')
Protein = models.CharField(max_length=255, unique=False, blank=True, null=True) Protein = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='蛋白質')
Total_fat = models.CharField(max_length=255, unique=False, blank=True, null=True) Total_fat = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='總脂肪')
Total_Carbohydrate = models.CharField(max_length=255, unique=False, blank=True, null=True) Total_Carbohydrate = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='總碳水化合物')
Total_sugar = models.CharField(max_length=255, unique=False, blank=True, null=True) Total_sugar = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='總糖分')
Dietary_fiber = models.CharField(max_length=255, unique=False, blank=True, null=True) Dietary_fiber = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='膳食纖維')
Soluble_fiber = models.CharField(max_length=255, unique=False, blank=True, null=True) Soluble_fiber = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='水溶性纖維')
Insoluble_fiber = models.CharField(max_length=255, unique=False, blank=True, null=True) Insoluble_fiber = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='非水溶性纖維')
K = models.CharField(max_length=255, unique=False, blank=True, null=True) K = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='鉀')
Ca = models.CharField(max_length=255, unique=False, blank=True, null=True) Ca = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='鈣')
Mg = models.CharField(max_length=255, unique=False, blank=True, null=True) Mg = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='鎂')
P = models.CharField(max_length=255, unique=False, blank=True, null=True) P = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='磷')
Fe = models.CharField(max_length=255, unique=False, blank=True, null=True) Fe = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='鐵')
Zn = models.CharField(max_length=255, unique=False, blank=True, null=True) Zn = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='鋅')
I = models.CharField(max_length=255, unique=False, blank=True, null=True) I = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='碘')
Cholesterol = models.CharField(max_length=255, unique=False, blank=True, null=True) Cholesterol = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='膽固醇')
Vitamin_B1 = models.CharField(max_length=255, unique=False, blank=True, null=True) Vitamin_B1 = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='維生素B1')
Vitamin_B2 = models.CharField(max_length=255, unique=False, blank=True, null=True) Vitamin_B2 = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='維生素B2')
Vitamin_C = models.CharField(max_length=255, unique=False, blank=True, null=True) Vitamin_C = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='維生素C')
Vitamin_B6 = models.CharField(max_length=255, unique=False, blank=True, null=True) Vitamin_B6 = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='維生素B6')
Vitamin_B12 = models.CharField(max_length=255, unique=False, blank=True, null=True) Vitamin_B12 = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='維生素B12')
Folate = models.CharField(max_length=255, unique=False, blank=True, null=True) Folate = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='葉酸')
Vitamin_A = models.CharField(max_length=255, unique=False, blank=True, null=True) Vitamin_A = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='維生素A')
Vitamin_D = models.CharField(max_length=255, unique=False, blank=True, null=True) Vitamin_D = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='維生素D')
Vitamin_K = models.CharField(max_length=255, unique=False, blank=True, null=True) Vitamin_K = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='維生素K')
Vitamin_E = models.CharField(max_length=255, unique=False, blank=True, null=True) Vitamin_E = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='維生素E')
Saturated_fatty_acid = models.CharField(max_length=255, unique=False, blank=True, null=True) Saturated_fatty_acid = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='飽和脂肪酸')
Monounsaturated_fatty_acid = models.CharField(max_length=255, unique=False, blank=True, null=True) Monounsaturated_fatty_acid = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='單元不飽和脂肪酸')
Polyunsaturated_fatty_acid = models.CharField(max_length=255, unique=False, blank=True, null=True) Polyunsaturated_fatty_acid = models.CharField(max_length=255, unique=False, blank=True, null=True, help_text='多元不飽和脂肪酸')
Ingredients = models.TextField() Ingredients = models.TextField(help_text='食材')
Steps = models.JSONField() Steps = models.JSONField(help_text='製作步驟')
Step_images_Base64 = models.JSONField() Step_images_Base64 = models.JSONField(help_text='製作步驟圖片')

366
Recipe/views.py
Unescape View File

@ -1,10 +1,8 @@
import asyncio import asyncio
import base64 import base64
import pandas as pd
from pandas import DataFrame
from playwright.async_api import Playwright, async_playwright, Error from playwright.async_api import Playwright, async_playwright, Error
from asgiref.sync import sync_to_async from asgiref.sync import sync_to_async
import time
from django.http import HttpResponseBadRequest, HttpResponse from django.http import HttpResponseBadRequest, HttpResponse
from django.views import View from django.views import View
from Recipe.models import Dish from Recipe.models import Dish
@ -22,14 +20,14 @@ def sync_main():
asyncio.run(main()) asyncio.run(main())
async def save_to_db(dish_data): async def save_to_db(dDish_data):
# 将异步ORM操作转换为同步，以适应Django ORM # 将异步ORM操作转换为同步，以适应Django ORM
dish, created = await sync_to_async(Dish.objects.update_or_create)( dish, created = await sync_to_async(Dish.objects.update_or_create)(
name=dish_data['name'], name= dDish_data['name'],
defaults=dish_data defaults= dDish_data
) )
action = "added" if created else "updated" sAction = "added" if created else "updated"
print(f"Dish '{dish.name}' was {action}.") print(f"Dish '{dish.name}' was {sAction}.")
# 將圖片轉為base64 # 將圖片轉為base64
@ -39,8 +37,8 @@ async def fetch_image_as_base64(page, image_url):
response = await page.request.get(image_url) response = await page.request.get(image_url)
if response.ok: if response.ok:
image_data = await response.body() image_data = await response.body()
image_base64 = base64.b64encode(image_data).decode() sImage_base64 = base64.b64encode(image_data).decode()
return image_base64 return sImage_base64
except Exception as e: except Exception as e:
print(f"Error fetching image: {e}") print(f"Error fetching image: {e}")
return None return None
@ -62,21 +60,19 @@ async def run(playwright: Playwright):
max_retries = 3
while True: while True:
# 訂位到所有匹配連結 # 訂位到所有匹配連結
links = page.locator('a.p-recipe-list-item__title-link') links = page.locator('a.p-recipe-list-item__title-link')
# 獲取連結數量 # 獲取連結數量
link_count = await links.count() iLink_count = await links.count()
# 點擊每個連結 # 點擊每個連結
for i in range(link_count): for i in range(iLink_count):
retry_count = 0 # 设置重试次数计数器 iRetry_count = 0 # 设置重试次数计数器
while retry_count < 3: # 假设最多重试3次 while iRetry_count < 3: # 假设最多重试3次
# 使用 nth(i) 定位第 i 個元素，並點擊
# 使用 nth(i) 定位第 i 個元素，並點擊
try: try:
await page.locator('a.p-recipe-list-item__title-link').nth(i).click() await page.locator('a.p-recipe-list-item__title-link').nth(i).click()
@ -86,229 +82,229 @@ async def run(playwright: Playwright):
print("------菜名-----") print("------菜名-----")
# 輸出名稱 # 輸出名稱
dishname = await page.text_content('.p-recipe-detail__title') sDishname = await page.text_content('.p-recipe-detail__title')
print(dishname) print(sDishname)
dishname_clean = dishname.strip().replace('\n', '') sDishname_clean = sDishname.strip().replace('\n', '')
# 菜名圖片 # 菜名圖片
image_element = page.locator('.p-recipe-detail__photo-image--pc-only') image_element = page.locator('.p-recipe-detail__photo-image--pc-only')
# 从元素的 'src' 属性中获取图片的 URL # 从元素的 'src' 属性中获取图片的 URL
image_url = await image_element.get_attribute('src') sImage_url = await image_element.get_attribute('src')
# 确保获取到的 URL 不为空 # 确保获取到的 URL 不为空
if image_url: if sImage_url:
# 获取图片的 Base64 编码 # 获取图片的 Base64 编码
Dish_image_base64 = await fetch_image_as_base64(page, image_url) sDish_image_base64 = await fetch_image_as_base64(page, sImage_url)
else: else:
Dish_image_base64 = None sDish_image_base64 = None
# print("------按讚數-----") # print("------按讚數-----")
# 輸出按讚數 # 輸出按讚數
Like_count = await page.text_content('.c-button-circle__top-text') sLike_count = await page.text_content('.c-button-circle__top-text')
# print(Likes_count) # print(Likes_count)
Likes_count_clean = Like_count.strip().replace('\n', '') sLikes_count_clean = sLike_count.strip().replace('\n', '')
# print("------標籤-----") # print("------標籤-----")
# 輸出標籤 # 輸出標籤
tags = await page.locator('.c-button-round-tag__link').all_text_contents() lTags = await page.locator('.c-button-round-tag__link').all_text_contents()
# for tag in tags: # for tag in tags:
# print(tag) # print(tag)
tags_clean = [tag.strip() for tag in tags] lTags_clean = [sTag.strip() for sTag in lTags]
# print("------適應症-----") # print("------適應症-----")
# 輸出適應症 # 輸出適應症
Indications = await page.locator('.c-recipes-relevant-dietary-concerns__text').all_text_contents() lIndications = await page.locator('.c-recipes-relevant-dietary-concerns__text').all_text_contents()
# for Indication in Indications: # for Indication in Indications:
# print(Indication) # print(Indication)
Indications_clean = [Indication.strip() for Indication in Indications] lIndications_clean = [sIndication.strip() for sIndication in lIndications]
# print("------營養標示-----") # print("------營養標示-----")
# 輸出營養標示 # 輸出營養標示
# 各營養標示分類 # 各營養標示分類
Calorie = '' sCalorie = ''
Salt = '' sSalt = ''
Protein = '' sProtein = ''
Fat = '' sFat = ''
Carbohydrate = '' sCarbohydrate = ''
Sugar = '' sSugar = ''
Dietary_fiber = '' sDietary_fiber = ''
Soluble_fiber = '' sSoluble_fiber = ''
Insoluble_fiber = '' sInsoluble_fiber = ''
Potassium = '' sPotassium = ''
Calcium = '' sCalcium = ''
Magnesium = '' sMagnesium = ''
Phosphorous = '' sPhosphorous = ''
Iron = '' sIron = ''
Zinc = '' sZinc = ''
Iodine = '' sIodine = ''
Cholesterol = '' sCholesterol = ''
Vitamin_B1 = '' sVitamin_B1 = ''
Vitamin_B2 = '' sVitamin_B2 = ''
Vitamin_C = '' sVitamin_C = ''
Vitamin_B6 = '' sVitamin_B6 = ''
Vitamin_B12 = '' sVitamin_B12 = ''
Folate = '' sFolate = ''
Vitamin_A = '' sVitamin_A = ''
Vitamin_D = '' sVitamin_D = ''
Vitamin_K = '' sVitamin_K = ''
Vitamin_E = '' sVitamin_E = ''
Saturated_fatty_acid = '' sSaturated_fatty_acid = ''
Monounsaturated_fatty_acid = '' sMonounsaturated_fatty_acid = ''
Polyunsaturated_fatty_acid = '' sPolyunsaturated_fatty_acid = ''
nutritions1 = await page.locator('.c-nutrition-table__cell--1').all_text_contents() lNutritions1 = await page.locator('.c-nutrition-table__cell--1').all_text_contents()
# for nutrition in nutritions1: # for nutrition in nutritions1:
# print(nutrition) # print(nutrition)
# nutritions_clean = [nutrition.strip().replace('\n', '') for nutrition in nutritions] # nutritions_clean = [nutrition.strip().replace('\n', '') for nutrition in nutritions]
for nutrition1 in nutritions1: for sNutrition1 in lNutritions1:
if 'エネルギー' in nutrition1: if 'エネルギー' in sNutrition1:
Calorie = nutrition1.split('エネルギー')[1].strip().replace('\n', '') sCalorie = sNutrition1.split('エネルギー')[1].strip().replace('\n', '')
if '食塩相当量' in nutrition1: if '食塩相当量' in sNutrition1:
Salt = nutrition1.split('食塩相当量')[1].strip().replace('\n', '') sSalt = sNutrition1.split('食塩相当量')[1].strip().replace('\n', '')
if 'たんぱく質' in nutrition1: if 'たんぱく質' in sNutrition1:
Protein = nutrition1.split('たんぱく質')[1].strip().replace('\n', '') sProtein = sNutrition1.split('たんぱく質')[1].strip().replace('\n', '')
if '脂質' in nutrition1: if '脂質' in sNutrition1:
Fat = nutrition1.split('脂質')[1].strip().replace('\n', '') sFat = sNutrition1.split('脂質')[1].strip().replace('\n', '')
if '炭水化物' in nutrition1: if '炭水化物' in sNutrition1:
Carbohydrate = nutrition1.split('炭水化物')[1].strip().replace('\n', '') sCarbohydrate = sNutrition1.split('炭水化物')[1].strip().replace('\n', '')
if '糖質' in nutrition1: if '糖質' in sNutrition1:
Sugar = nutrition1.split('糖質')[1].strip().replace('\n', '') sSugar = sNutrition1.split('糖質')[1].strip().replace('\n', '')
if '食物繊維' in nutrition1: if '食物繊維' in sNutrition1:
Dietary_fiber = nutrition1.split('食物繊維')[1].strip().replace('\n', '') sDietary_fiber = sNutrition1.split('食物繊維')[1].strip().replace('\n', '')
if '水溶性食物繊維' in nutrition1: if '水溶性食物繊維' in sNutrition1:
Soluble_fiber = nutrition1.split('水溶性食物繊維')[1].strip().replace('\n', '') sSoluble_fiber = sNutrition1.split('水溶性食物繊維')[1].strip().replace('\n', '')
if '不溶性食物繊維' in nutrition1: if '不溶性食物繊維' in sNutrition1:
Insoluble_fiber = nutrition1.split('不溶性食物繊維')[1].strip().replace('\n', '') sInsoluble_fiber = sNutrition1.split('不溶性食物繊維')[1].strip().replace('\n', '')
if 'カリウム' in nutrition1: if 'カリウム' in sNutrition1:
Potassium = nutrition1.split('カリウム')[1].strip().replace('\n', '') sPotassium = sNutrition1.split('カリウム')[1].strip().replace('\n', '')
nutritions2 = await page.locator('.c-nutrition-table__cell--2').all_text_contents() lNutritions2 = await page.locator('.c-nutrition-table__cell--2').all_text_contents()
for nutrition2 in nutritions2: for sNutrition2 in lNutritions2:
if 'カルシウム' in nutrition2: if 'カルシウム' in sNutrition2:
Calcium = nutrition2.split('カルシウム')[1].strip().replace('\n', '') sCalcium = sNutrition2.split('カルシウム')[1].strip().replace('\n', '')
if 'マグネシウム' in nutrition2: if 'マグネシウム' in sNutrition2:
Magnesium = nutrition2.split('マグネシウム')[1].strip().replace('\n', '') sMagnesium = sNutrition2.split('マグネシウム')[1].strip().replace('\n', '')
if 'リン' in nutrition2: if 'リン' in sNutrition2:
Phosphorous = nutrition2.split('リン')[1].strip().replace('\n', '') sPhosphorous = sNutrition2.split('リン')[1].strip().replace('\n', '')
if '鉄' in nutrition2: if '鉄' in sNutrition2:
Iron = nutrition2.split('鉄')[1].strip().replace('\n', '') sIron = sNutrition2.split('鉄')[1].strip().replace('\n', '')
if '亜鉛' in nutrition2: if '亜鉛' in sNutrition2:
Zinc = nutrition2.split('亜鉛')[1].strip().replace('\n', '') sZinc = sNutrition2.split('亜鉛')[1].strip().replace('\n', '')
if 'ヨウ素' in nutrition2: if 'ヨウ素' in sNutrition2:
Iodine = nutrition2.split('ヨウ素')[1].strip().replace('\n', '') sIodine = sNutrition2.split('ヨウ素')[1].strip().replace('\n', '')
if 'コレステロール' in nutrition2: if 'コレステロール' in sNutrition2:
Cholesterol = nutrition2.split('コレステロール')[1].strip().replace('\n', '') sCholesterol = sNutrition2.split('コレステロール')[1].strip().replace('\n', '')
if 'ビタミンB1' in nutrition2: if 'ビタミンB1' in sNutrition2:
Vitamin_B1 = nutrition2.split('ビタミンB1')[1].strip().replace('\n', '') sVitamin_B1 = sNutrition2.split('ビタミンB1')[1].strip().replace('\n', '')
if 'ビタミンB2' in nutrition2: if 'ビタミンB2' in sNutrition2:
Vitamin_B2 = nutrition2.split('ビタミンB2')[1].strip().replace('\n', '') sVitamin_B2 = sNutrition2.split('ビタミンB2')[1].strip().replace('\n', '')
if 'ビタミンC' in nutrition2: if 'ビタミンC' in sNutrition2:
Vitamin_C = nutrition2.split('ビタミンC')[1].strip().replace('\n', '') sVitamin_C = sNutrition2.split('ビタミンC')[1].strip().replace('\n', '')
nutritions3 = await page.locator('.c-nutrition-table__cell--3').all_text_contents() lNutritions3 = await page.locator('.c-nutrition-table__cell--3').all_text_contents()
for nutrition3 in nutritions3: for sNutrition3 in lNutritions3:
if 'ビタミンB6' in nutrition3: if 'ビタミンB6' in sNutrition3:
Vitamin_B6 = nutrition3.split('ビタミンB6')[1].strip().replace('\n', '') sVitamin_B6 = sNutrition3.split('ビタミンB6')[1].strip().replace('\n', '')
if 'ビタミンB12' in nutrition3: if 'ビタミンB12' in sNutrition3:
Vitamin_B12 = nutrition3.split('ビタミンB12')[1].strip().replace('\n', '') sVitamin_B12 = sNutrition3.split('ビタミンB12')[1].strip().replace('\n', '')
if '葉酸' in nutrition3: if '葉酸' in sNutrition3:
Folate = nutrition3.split('葉酸')[1].strip().replace('\n', '') sFolate = sNutrition3.split('葉酸')[1].strip().replace('\n', '')
if 'ビタミンA' in nutrition3: if 'ビタミンA' in sNutrition3:
Vitamin_A = nutrition3.split('ビタミンA')[1].strip().replace('\n', '') sVitamin_A = sNutrition3.split('ビタミンA')[1].strip().replace('\n', '')
if 'ビタミンD' in nutrition3: if 'ビタミンD' in sNutrition3:
Vitamin_D = nutrition3.split('ビタミンD')[1].strip().replace('\n', '') sVitamin_D = sNutrition3.split('ビタミンD')[1].strip().replace('\n', '')
if 'ビタミンK' in nutrition3: if 'ビタミンK' in sNutrition3:
Vitamin_K = nutrition3.split('ビタミンK')[1].strip().replace('\n', '') sVitamin_K = sNutrition3.split('ビタミンK')[1].strip().replace('\n', '')
if 'ビタミンE' in nutrition3: if 'ビタミンE' in sNutrition3:
Vitamin_E = nutrition3.split('ビタミンE')[1].strip().replace('\n', '') sVitamin_E = sNutrition3.split('ビタミンE')[1].strip().replace('\n', '')
if '飽和脂肪酸' in nutrition3: if '飽和脂肪酸' in sNutrition3:
Saturated_fatty_acid = nutrition3.split('飽和脂肪酸')[1].strip().replace('\n', '') sSaturated_fatty_acid = sNutrition3.split('飽和脂肪酸')[1].strip().replace('\n', '')
if '一価不飽和脂肪酸' in nutrition3: if '一価不飽和脂肪酸' in sNutrition3:
Monounsaturated_fatty_acid = nutrition3.split('一価不飽和脂肪酸')[1].strip().replace('\n', '') sMonounsaturated_fatty_acid = sNutrition3.split('一価不飽和脂肪酸')[1].strip().replace('\n', '')
if '多価不飽和脂肪酸' in nutrition3: if '多価不飽和脂肪酸' in sNutrition3:
Polyunsaturated_fatty_acid = nutrition3.split('多価不飽和脂肪酸')[1].strip().replace('\n', '') sPolyunsaturated_fatty_acid = sNutrition3.split('多価不飽和脂肪酸')[1].strip().replace('\n', '')
# 食料 # 食料
Ingredients = await page.locator('.p-recipe-ingredient-list__item').all_text_contents() lIngredients = await page.locator('.p-recipe-ingredient-list__item').all_text_contents()
# for Ingredient in Ingredients: # for Ingredient in Ingredients:
# print(Ingredient) # print(Ingredient)
Ingredients_clean = [Ingredient.strip().replace('\n', '') for Ingredient in Ingredients] lIngredients_clean = [sIngredient.strip().replace('\n', '') for sIngredient in lIngredients]
print('------作法步驟-----') print('------作法步驟-----')
# 作法 # 作法
Steps = await page.locator('.p-recipe-step__item').all_text_contents() lSteps = await page.locator('.p-recipe-step__item').all_text_contents()
# for Step in Steps: # for Step in Steps:
# print(Step) # print(Step)
Steps_clean = [Step.strip().replace('\n', '') for Step in Steps] lSteps_clean = [sStep.strip().replace('\n', '') for sStep in lSteps]
# 定位到所有步骤的图片元素 # 定位到所有步骤的图片元素
image_elements = page.locator('.p-recipe-step__item-image') Image_elements = page.locator('.p-recipe-step__item-image')
# 获取所有图片元素的 src 属性（即图片的 URL） # 获取所有图片元素的 src 属性（即图片的 URL）
image_urls = await image_elements.evaluate_all("elements => elements.map(e => e.getAttribute('src'))") lImage_urls = await Image_elements.evaluate_all("elements => elements.map(e => e.getAttribute('src'))")
# 遍历图片 URL 列表，下载图片并转换为 Base64 # 遍历图片 URL 列表，下载图片并转换为 Base64
Step_images_base64 = [] lStep_images_base64 = []
for image_url in image_urls: for sImage_url in lImage_urls:
# 直接使用图片的 URL 下载图片并转换为 Base64 # 直接使用图片的 URL 下载图片并转换为 Base64
image_base64 = await fetch_image_as_base64(page, image_url) sImage_base64 = await fetch_image_as_base64(page, sImage_url)
if image_base64: if sImage_base64:
Step_images_base64.append(image_base64) lStep_images_base64.append(sImage_base64)
dish_data = { dDish_data = {
'name': dishname_clean, 'name': sDishname_clean,
'image': Dish_image_base64, 'image': sDish_image_base64,
'likes': Likes_count_clean, 'likes': sLikes_count_clean,
'tags': ", ".join(tags_clean), 'tags': ", ".join(lTags_clean),
'indications': ", ".join(Indications_clean), 'indications': ", ".join(lIndications_clean),
'Calories': Calorie, 'Calories': sCalorie,
'Salt': Salt, 'Salt': sSalt,
'Protein': Protein, 'Protein': sProtein,
'Total_fat': Fat, 'Total_fat': sFat,
'Total_Carbohydrate': Carbohydrate, 'Total_Carbohydrate': sCarbohydrate,
'Total_sugar': Sugar, 'Total_sugar': sSugar,
'Dietary_fiber': Dietary_fiber, 'Dietary_fiber': sDietary_fiber,
'Soluble_fiber': Soluble_fiber, 'Soluble_fiber': sSoluble_fiber,
'Insoluble_fiber': Insoluble_fiber, 'Insoluble_fiber': sInsoluble_fiber,
'K': Potassium, 'K': sPotassium,
'Ca': Calcium, 'Ca': sCalcium,
'Mg': Magnesium, 'Mg': sMagnesium,
'P': Phosphorous, 'P': sPhosphorous,
'Fe': Iron, 'Fe': sIron,
'Zn': Zinc, 'Zn': sZinc,
'I': Iodine, 'I': sIodine,
'Cholesterol': Cholesterol, 'Cholesterol': sCholesterol,
'Vitamin_B1': Vitamin_B1, 'Vitamin_B1': sVitamin_B1,
'Vitamin_B2': Vitamin_B2, 'Vitamin_B2': sVitamin_B2,
'Vitamin_C': Vitamin_C, 'Vitamin_C': sVitamin_C,
'Vitamin_B6': Vitamin_B6, 'Vitamin_B6': sVitamin_B6,
'Vitamin_B12': Vitamin_B12, 'Vitamin_B12': sVitamin_B12,
'Folate': Folate, 'Folate': sFolate,
'Vitamin_A': Vitamin_A, 'Vitamin_A': sVitamin_A,
'Vitamin_D': Vitamin_D, 'Vitamin_D': sVitamin_D,
'Vitamin_K': Vitamin_K, 'Vitamin_K': sVitamin_K,
'Vitamin_E': Vitamin_E, 'Vitamin_E': sVitamin_E,
'Saturated_fatty_acid': Saturated_fatty_acid, 'Saturated_fatty_acid': sSaturated_fatty_acid,
'Monounsaturated_fatty_acid': Monounsaturated_fatty_acid, 'Monounsaturated_fatty_acid': sMonounsaturated_fatty_acid,
'Polyunsaturated_fatty_acid': Polyunsaturated_fatty_acid, 'Polyunsaturated_fatty_acid': sPolyunsaturated_fatty_acid,
'Ingredients': ", ".join(Ingredients_clean), 'Ingredients': ", ".join(lIngredients_clean),
'Steps': Steps_clean, 'Steps': lSteps_clean,
'Step_images_Base64': Step_images_base64 'Step_images_Base64': lStep_images_base64
} }
await save_to_db(dish_data) await save_to_db(dDish_data)
break break
except Exception as e: # 捕获可能发生的异常 except Exception as e: # 捕获可能发生的异常
print(f"遇到错误：{e}，尝试返回并重试") print(f"遇到错误：{e}，尝试返回并重试")
await page.go_back() # 返回前一页 await page.go_back() # 返回前一页
retry_count += 1 # 重试计数器加1 iRetry_count += 1 # 重试计数器加1
if retry_count >= 3: if iRetry_count >= 3:
print("重试次数超限，跳过当前链接") print("重试次数超限，跳过当前链接")
break # 跳出循环，处理下一个链接 break # 跳出循环，处理下一个链接
# 使用浏览器的后退功能返回列表页，这样不需要重新加载初始URL # 使用浏览器的后退功能返回列表页，这样不需要重新加载初始URL

Write Preview
Loading…
Cancel
Save