332 lines
16 KiB
Python
332 lines
16 KiB
Python
import asyncio
|
||
import base64
|
||
|
||
from playwright.async_api import Playwright, async_playwright, Error
|
||
from asgiref.sync import sync_to_async
|
||
from django.http import HttpResponseBadRequest, HttpResponse
|
||
from django.views import View
|
||
from Recipe.models import Dish
|
||
|
||
|
||
class RecipeCrawlerView(View):
|
||
def get(self, request):
|
||
sync_main()
|
||
return HttpResponse("Success!", content_type="text/plain")
|
||
|
||
|
||
# 因應django ORM，以同步函數執行非同步程式
|
||
def sync_main():
|
||
# 使用sync_to_async包装器調用非同步函數
|
||
asyncio.run(main())
|
||
|
||
|
||
async def save_to_db(dDish_data):
|
||
# 将异步ORM操作转换为同步，以适应Django ORM
|
||
dish, created = await sync_to_async(Dish.objects.update_or_create)(
|
||
name= dDish_data['name'],
|
||
defaults= dDish_data
|
||
)
|
||
sAction = "added" if created else "updated"
|
||
print(f"Dish '{dish.name}' was {sAction}.")
|
||
|
||
|
||
# 將圖片轉為base64
|
||
async def fetch_image_as_base64(page, image_url):
|
||
print("Fetching image from URL:", image_url)
|
||
try:
|
||
response = await page.request.get(image_url)
|
||
if response.ok:
|
||
image_data = await response.body()
|
||
sImage_base64 = base64.b64encode(image_data).decode()
|
||
return sImage_base64
|
||
except Exception as e:
|
||
print(f"Error fetching image: {e}")
|
||
return None
|
||
|
||
|
||
async def run(playwright: Playwright):
|
||
browser = await playwright.chromium.launch(headless=False)
|
||
context = await browser.new_context()
|
||
page = await context.new_page()
|
||
await page.goto("https://oishi-kenko.com/recipes")
|
||
await page.get_by_role("link", name="ログイン").click()
|
||
await page.get_by_role("link", name="メールアドレス でログイン").click()
|
||
await page.locator("#secure_account_credential_email").click()
|
||
await page.locator("#secure_account_credential_email").fill("asd851117005545@gmail.com")
|
||
await page.locator("#secure_account_credential_email").press("Tab")
|
||
await page.locator("#secure_account_credential_password").fill("a22897051")
|
||
await page.get_by_role("button", name="ログイン").click()
|
||
await page.goto("https://oishi-kenko.com/recipes")
|
||
|
||
|
||
|
||
while True:
|
||
# 訂位到所有匹配連結
|
||
links = page.locator('a.p-recipe-list-item__title-link')
|
||
|
||
# 獲取連結數量
|
||
iLink_count = await links.count()
|
||
|
||
# 點擊每個連結
|
||
for i in range(iLink_count):
|
||
iRetry_count = 0 # 设置重试次数计数器
|
||
while iRetry_count < 3: # 假设最多重试3次
|
||
|
||
# 使用 nth(i) 定位第 i 個元素，並點擊
|
||
try:
|
||
await page.locator('a.p-recipe-list-item__title-link').nth(i).click()
|
||
|
||
# 等待頁面
|
||
await page.wait_for_load_state('networkidle')
|
||
# await asyncio.sleep(5)
|
||
|
||
print("------菜名-----")
|
||
# 輸出名稱
|
||
sDishname = await page.text_content('.p-recipe-detail__title')
|
||
print(sDishname)
|
||
sDishname_clean = sDishname.strip().replace('\n', '')
|
||
|
||
# 菜名圖片
|
||
image_element = page.locator('.p-recipe-detail__photo-image--pc-only')
|
||
|
||
# 从元素的 'src' 属性中获取图片的 URL
|
||
sImage_url = await image_element.get_attribute('src')
|
||
|
||
# 确保获取到的 URL 不为空
|
||
if sImage_url:
|
||
# 获取图片的 Base64 编码
|
||
sDish_image_base64 = await fetch_image_as_base64(page, sImage_url)
|
||
else:
|
||
sDish_image_base64 = None
|
||
|
||
# print("------按讚數-----")
|
||
# 輸出按讚數
|
||
sLike_count = await page.text_content('.c-button-circle__top-text')
|
||
# print(Likes_count)
|
||
sLikes_count_clean = sLike_count.strip().replace('\n', '')
|
||
|
||
# print("------標籤-----")
|
||
# 輸出標籤
|
||
lTags = await page.locator('.c-button-round-tag__link').all_text_contents()
|
||
# for tag in tags:
|
||
# print(tag)
|
||
lTags_clean = [sTag.strip() for sTag in lTags]
|
||
|
||
# print("------適應症-----")
|
||
# 輸出適應症
|
||
lIndications = await page.locator('.c-recipes-relevant-dietary-concerns__text').all_text_contents()
|
||
# for Indication in Indications:
|
||
# print(Indication)
|
||
lIndications_clean = [sIndication.strip() for sIndication in lIndications]
|
||
|
||
# print("------營養標示-----")
|
||
# 輸出營養標示
|
||
|
||
# 各營養標示分類
|
||
sCalorie = ''
|
||
sSalt = ''
|
||
sProtein = ''
|
||
sFat = ''
|
||
sCarbohydrate = ''
|
||
sSugar = ''
|
||
sDietary_fiber = ''
|
||
sSoluble_fiber = ''
|
||
sInsoluble_fiber = ''
|
||
sPotassium = ''
|
||
sCalcium = ''
|
||
sMagnesium = ''
|
||
sPhosphorous = ''
|
||
sIron = ''
|
||
sZinc = ''
|
||
sIodine = ''
|
||
sCholesterol = ''
|
||
sVitamin_B1 = ''
|
||
sVitamin_B2 = ''
|
||
sVitamin_C = ''
|
||
sVitamin_B6 = ''
|
||
sVitamin_B12 = ''
|
||
sFolate = ''
|
||
sVitamin_A = ''
|
||
sVitamin_D = ''
|
||
sVitamin_K = ''
|
||
sVitamin_E = ''
|
||
sSaturated_fatty_acid = ''
|
||
sMonounsaturated_fatty_acid = ''
|
||
sPolyunsaturated_fatty_acid = ''
|
||
|
||
lNutritions1 = await page.locator('.c-nutrition-table__cell--1').all_text_contents()
|
||
# for nutrition in nutritions1:
|
||
# print(nutrition)
|
||
# nutritions_clean = [nutrition.strip().replace('\n', '') for nutrition in nutritions]
|
||
for sNutrition1 in lNutritions1:
|
||
if 'エネルギー' in sNutrition1:
|
||
sCalorie = sNutrition1.split('エネルギー')[1].strip().replace('\n', '')
|
||
if '食塩相当量' in sNutrition1:
|
||
sSalt = sNutrition1.split('食塩相当量')[1].strip().replace('\n', '')
|
||
if 'たんぱく質' in sNutrition1:
|
||
sProtein = sNutrition1.split('たんぱく質')[1].strip().replace('\n', '')
|
||
if '脂質' in sNutrition1:
|
||
sFat = sNutrition1.split('脂質')[1].strip().replace('\n', '')
|
||
if '炭水化物' in sNutrition1:
|
||
sCarbohydrate = sNutrition1.split('炭水化物')[1].strip().replace('\n', '')
|
||
if '糖質' in sNutrition1:
|
||
sSugar = sNutrition1.split('糖質')[1].strip().replace('\n', '')
|
||
if '食物繊維' in sNutrition1:
|
||
sDietary_fiber = sNutrition1.split('食物繊維')[1].strip().replace('\n', '')
|
||
if '水溶性食物繊維' in sNutrition1:
|
||
sSoluble_fiber = sNutrition1.split('水溶性食物繊維')[1].strip().replace('\n', '')
|
||
if '不溶性食物繊維' in sNutrition1:
|
||
sInsoluble_fiber = sNutrition1.split('不溶性食物繊維')[1].strip().replace('\n', '')
|
||
if 'カリウム' in sNutrition1:
|
||
sPotassium = sNutrition1.split('カリウム')[1].strip().replace('\n', '')
|
||
|
||
lNutritions2 = await page.locator('.c-nutrition-table__cell--2').all_text_contents()
|
||
|
||
for sNutrition2 in lNutritions2:
|
||
if 'カルシウム' in sNutrition2:
|
||
sCalcium = sNutrition2.split('カルシウム')[1].strip().replace('\n', '')
|
||
if 'マグネシウム' in sNutrition2:
|
||
sMagnesium = sNutrition2.split('マグネシウム')[1].strip().replace('\n', '')
|
||
if 'リン' in sNutrition2:
|
||
sPhosphorous = sNutrition2.split('リン')[1].strip().replace('\n', '')
|
||
if '鉄' in sNutrition2:
|
||
sIron = sNutrition2.split('鉄')[1].strip().replace('\n', '')
|
||
if '亜鉛' in sNutrition2:
|
||
sZinc = sNutrition2.split('亜鉛')[1].strip().replace('\n', '')
|
||
if 'ヨウ素' in sNutrition2:
|
||
sIodine = sNutrition2.split('ヨウ素')[1].strip().replace('\n', '')
|
||
if 'コレステロール' in sNutrition2:
|
||
sCholesterol = sNutrition2.split('コレステロール')[1].strip().replace('\n', '')
|
||
if 'ビタミンB1' in sNutrition2:
|
||
sVitamin_B1 = sNutrition2.split('ビタミンB1')[1].strip().replace('\n', '')
|
||
if 'ビタミンB2' in sNutrition2:
|
||
sVitamin_B2 = sNutrition2.split('ビタミンB2')[1].strip().replace('\n', '')
|
||
if 'ビタミンC' in sNutrition2:
|
||
sVitamin_C = sNutrition2.split('ビタミンC')[1].strip().replace('\n', '')
|
||
|
||
lNutritions3 = await page.locator('.c-nutrition-table__cell--3').all_text_contents()
|
||
|
||
for sNutrition3 in lNutritions3:
|
||
if 'ビタミンB6' in sNutrition3:
|
||
sVitamin_B6 = sNutrition3.split('ビタミンB6')[1].strip().replace('\n', '')
|
||
if 'ビタミンB12' in sNutrition3:
|
||
sVitamin_B12 = sNutrition3.split('ビタミンB12')[1].strip().replace('\n', '')
|
||
if '葉酸' in sNutrition3:
|
||
sFolate = sNutrition3.split('葉酸')[1].strip().replace('\n', '')
|
||
if 'ビタミンA' in sNutrition3:
|
||
sVitamin_A = sNutrition3.split('ビタミンA')[1].strip().replace('\n', '')
|
||
if 'ビタミンD' in sNutrition3:
|
||
sVitamin_D = sNutrition3.split('ビタミンD')[1].strip().replace('\n', '')
|
||
if 'ビタミンK' in sNutrition3:
|
||
sVitamin_K = sNutrition3.split('ビタミンK')[1].strip().replace('\n', '')
|
||
if 'ビタミンE' in sNutrition3:
|
||
sVitamin_E = sNutrition3.split('ビタミンE')[1].strip().replace('\n', '')
|
||
if '飽和脂肪酸' in sNutrition3:
|
||
sSaturated_fatty_acid = sNutrition3.split('飽和脂肪酸')[1].strip().replace('\n', '')
|
||
if '一価不飽和脂肪酸' in sNutrition3:
|
||
sMonounsaturated_fatty_acid = sNutrition3.split('一価不飽和脂肪酸')[1].strip().replace('\n', '')
|
||
if '多価不飽和脂肪酸' in sNutrition3:
|
||
sPolyunsaturated_fatty_acid = sNutrition3.split('多価不飽和脂肪酸')[1].strip().replace('\n', '')
|
||
|
||
# 食料
|
||
lIngredients = await page.locator('.p-recipe-ingredient-list__item').all_text_contents()
|
||
# for Ingredient in Ingredients:
|
||
# print(Ingredient)
|
||
lIngredients_clean = [sIngredient.strip().replace('\n', '') for sIngredient in lIngredients]
|
||
|
||
print('------作法步驟-----')
|
||
|
||
# 作法
|
||
lSteps = await page.locator('.p-recipe-step__item').all_text_contents()
|
||
# for Step in Steps:
|
||
# print(Step)
|
||
lSteps_clean = [sStep.strip().replace('\n', '') for sStep in lSteps]
|
||
|
||
# 定位到所有步骤的图片元素
|
||
Image_elements = page.locator('.p-recipe-step__item-image')
|
||
|
||
# 获取所有图片元素的 src 属性（即图片的 URL）
|
||
lImage_urls = await Image_elements.evaluate_all("elements => elements.map(e => e.getAttribute('src'))")
|
||
|
||
# 遍历图片 URL 列表，下载图片并转换为 Base64
|
||
lStep_images_base64 = []
|
||
for sImage_url in lImage_urls:
|
||
# 直接使用图片的 URL 下载图片并转换为 Base64
|
||
sImage_base64 = await fetch_image_as_base64(page, sImage_url)
|
||
if sImage_base64:
|
||
lStep_images_base64.append(sImage_base64)
|
||
|
||
dDish_data = {
|
||
'name': sDishname_clean,
|
||
'image': sDish_image_base64,
|
||
'likes': sLikes_count_clean,
|
||
'tags': ", ".join(lTags_clean),
|
||
'indications': ", ".join(lIndications_clean),
|
||
'Calories': sCalorie,
|
||
'Salt': sSalt,
|
||
'Protein': sProtein,
|
||
'Total_fat': sFat,
|
||
'Total_Carbohydrate': sCarbohydrate,
|
||
'Total_sugar': sSugar,
|
||
'Dietary_fiber': sDietary_fiber,
|
||
'Soluble_fiber': sSoluble_fiber,
|
||
'Insoluble_fiber': sInsoluble_fiber,
|
||
'K': sPotassium,
|
||
'Ca': sCalcium,
|
||
'Mg': sMagnesium,
|
||
'P': sPhosphorous,
|
||
'Fe': sIron,
|
||
'Zn': sZinc,
|
||
'I': sIodine,
|
||
'Cholesterol': sCholesterol,
|
||
'Vitamin_B1': sVitamin_B1,
|
||
'Vitamin_B2': sVitamin_B2,
|
||
'Vitamin_C': sVitamin_C,
|
||
'Vitamin_B6': sVitamin_B6,
|
||
'Vitamin_B12': sVitamin_B12,
|
||
'Folate': sFolate,
|
||
'Vitamin_A': sVitamin_A,
|
||
'Vitamin_D': sVitamin_D,
|
||
'Vitamin_K': sVitamin_K,
|
||
'Vitamin_E': sVitamin_E,
|
||
'Saturated_fatty_acid': sSaturated_fatty_acid,
|
||
'Monounsaturated_fatty_acid': sMonounsaturated_fatty_acid,
|
||
'Polyunsaturated_fatty_acid': sPolyunsaturated_fatty_acid,
|
||
'Ingredients': ", ".join(lIngredients_clean),
|
||
'Steps': lSteps_clean,
|
||
'Step_images_Base64': lStep_images_base64
|
||
}
|
||
|
||
await save_to_db(dDish_data)
|
||
|
||
break
|
||
except Exception as e: # 捕获可能发生的异常
|
||
print(f"遇到错误：{e}，尝试返回并重试")
|
||
await page.go_back() # 返回前一页
|
||
iRetry_count += 1 # 重试计数器加1
|
||
if iRetry_count >= 3:
|
||
print("重试次数超限，跳过当前链接")
|
||
break # 跳出循环，处理下一个链接
|
||
# 使用浏览器的后退功能返回列表页，这样不需要重新加载初始URL
|
||
await page.go_back()
|
||
|
||
try:
|
||
await page.click('span.next a[rel="next"]')
|
||
await page.wait_for_load_state('networkidle')
|
||
except Error:
|
||
# 如果“下一頁”不存在，break
|
||
break
|
||
# ---------------------
|
||
await context.close()
|
||
await browser.close()
|
||
|
||
|
||
async def main() -> None:
|
||
async with async_playwright() as playwright:
|
||
await run(playwright)
|
||
|
||
# asyncio.run(main())
|
||
# async def simple_test():
|
||
# print("Simple async test")
|
||
#
|
||
# asyncio.run(simple_test()) |