RecipeCrawler/Recipe/views.py

import asyncio
import base64

from playwright.async_api import Playwright, async_playwright, Error
from asgiref.sync import sync_to_async
from django.http import HttpResponseBadRequest, HttpResponse
from django.views import View
from Recipe.models import Dish


class RecipeCrawlerView(View):
    def get(self, request):
        sync_main()
        return HttpResponse("Success!", content_type="text/plain")


# 因應django ORM，以同步函數執行非同步程式
def sync_main():
    # 使用sync_to_async包装器調用非同步函數
    asyncio.run(main())


async def save_to_db(dDish_data):
    # 将异步ORM操作转换为同步，以适应Django ORM
    oDish, bCreated = await sync_to_async(Dish.objects.update_or_create)(
        name= dDish_data['name'],
        defaults= dDish_data
    )
    sAction = "added" if bCreated else "updated"
    print(f"Dish '{oDish.name}' was {sAction}.")


# 將圖片轉為base64
async def fetch_image_as_base64(oPage, sImage_url):
    print("Fetching image from URL:", sImage_url)
    try:
        oResponse = await oPage.request.get(sImage_url)
        if oResponse.ok:
            bytImage_data = await oResponse.body()
            sImage_base64 = base64.b64encode(bytImage_data).decode()
            return sImage_base64
    except Exception as e:
        print(f"Error fetching image: {e}")
    return None


async def run(playwright: Playwright):
    oBrowser = await playwright.chromium.launch(headless=False)
    oContext = await oBrowser.new_context()
    oPage = await oContext.new_page()
    await oPage.goto("https://oishi-kenko.com/recipes")
    await oPage.get_by_role("link", name="ログイン").click()
    await oPage.get_by_role("link", name="メールアドレス でログイン").click()
    await oPage.locator("#secure_account_credential_email").click()
    await oPage.locator("#secure_account_credential_email").fill("asd851117005545@gmail.com")
    await oPage.locator("#secure_account_credential_email").press("Tab")
    await oPage.locator("#secure_account_credential_password").fill("a22897051")
    await oPage.get_by_role("button", name="ログイン").click()
    await oPage.goto("https://oishi-kenko.com/recipes")



    while True:
        # 訂位到所有匹配連結
        oLinks = oPage.locator('a.p-recipe-list-item__title-link')

        # 獲取連結數量
        iLink_count = await oLinks.count()

        # 點擊每個連結
        for i in range(iLink_count):
            iRetry_count = 0  # 设置重试次数计数器
            while iRetry_count < 5:  # 假设最多重试3次

                # 使用 nth(i) 定位第 i 個元素，並點擊
                try:
                    await oPage.locator('a.p-recipe-list-item__title-link').nth(i).click()

                    # 等待頁面
                    await oPage.wait_for_load_state('networkidle')
                    # await asyncio.sleep(5)

                    print("------菜名-----")
                    # 輸出名稱
                    sDishname = await oPage.text_content('.p-recipe-detail__title')
                    print(sDishname)
                    sDishname_clean = sDishname.strip().replace('\n', '')

                    # 菜名圖片
                    oImage_element = oPage.locator('.p-recipe-detail__photo-image--pc-only')

                    # 从元素的 'src' 属性中获取图片的 URL
                    sImage_url = await oImage_element.get_attribute('src')

                    # 确保获取到的 URL 不为空
                    if sImage_url:
                        # 获取图片的 Base64 编码
                        sDish_image_base64 = await fetch_image_as_base64(oPage, sImage_url)
                    else:
                        sDish_image_base64 = None

                    # print("------按讚數-----")
                    # 輸出按讚數
                    sLike_count = await oPage.text_content('.c-button-circle__top-text')
                    # print(Likes_count)
                    sLikes_count_clean = sLike_count.strip().replace('\n', '')

                    # print("------標籤-----")
                    # 輸出標籤
                    lTags = await oPage.locator('.c-button-round-tag__link').all_text_contents()
                    # for tag in tags:
                        # print(tag)
                    lTags_clean = [sTag.strip() for sTag in lTags]

                    # print("------適應症-----")
                    # 輸出適應症
                    lIndications = await oPage.locator('.c-recipes-relevant-dietary-concerns__text').all_text_contents()
                    # for Indication in Indications:
                    #     print(Indication)
                    lIndications_clean = [sIndication.strip() for sIndication in lIndications]

                    # print("------營養標示-----")
                    # 輸出營養標示

                    # 各營養標示分類
                    sCalorie = ''
                    sSalt = ''
                    sProtein = ''
                    sFat = ''
                    sCarbohydrate = ''
                    sSugar = ''
                    sDietary_fiber = ''
                    sSoluble_fiber = ''
                    sInsoluble_fiber = ''
                    sPotassium = ''
                    sCalcium = ''
                    sMagnesium = ''
                    sPhosphorous = ''
                    sIron = ''
                    sZinc = ''
                    sIodine = ''
                    sCholesterol = ''
                    sVitamin_B1 = ''
                    sVitamin_B2 = ''
                    sVitamin_C = ''
                    sVitamin_B6 = ''
                    sVitamin_B12 = ''
                    sFolate = ''
                    sVitamin_A = ''
                    sVitamin_D = ''
                    sVitamin_K = ''
                    sVitamin_E = ''
                    sSaturated_fatty_acid = ''
                    sMonounsaturated_fatty_acid = ''
                    sPolyunsaturated_fatty_acid = ''

                    lNutritions1 = await oPage.locator('.c-nutrition-table__cell--1').all_text_contents()
                    # for nutrition in nutritions1:
                    #     print(nutrition)
                    # nutritions_clean = [nutrition.strip().replace('\n', '') for nutrition in nutritions]
                    for sNutrition1 in lNutritions1:
                        if 'エネルギー' in sNutrition1:
                            sCalorie = sNutrition1.split('エネルギー')[1].strip().replace('\n', '')
                        if '食塩相当量' in sNutrition1:
                            sSalt = sNutrition1.split('食塩相当量')[1].strip().replace('\n', '')
                        if 'たんぱく質' in sNutrition1:
                            sProtein = sNutrition1.split('たんぱく質')[1].strip().replace('\n', '')
                        if '脂質' in sNutrition1:
                            sFat = sNutrition1.split('脂質')[1].strip().replace('\n', '')
                        if '炭水化物' in sNutrition1:
                            sCarbohydrate = sNutrition1.split('炭水化物')[1].strip().replace('\n', '')
                        if '糖質' in sNutrition1:
                            sSugar = sNutrition1.split('糖質')[1].strip().replace('\n', '')
                        if '食物繊維' in sNutrition1:
                            sDietary_fiber = sNutrition1.split('食物繊維')[1].strip().replace('\n', '')
                        if '水溶性食物繊維' in sNutrition1:
                            sSoluble_fiber = sNutrition1.split('水溶性食物繊維')[1].strip().replace('\n', '')
                        if '不溶性食物繊維' in sNutrition1:
                            sInsoluble_fiber = sNutrition1.split('不溶性食物繊維')[1].strip().replace('\n', '')
                        if 'カリウム' in sNutrition1:
                            sPotassium = sNutrition1.split('カリウム')[1].strip().replace('\n', '')

                    lNutritions2 = await oPage.locator('.c-nutrition-table__cell--2').all_text_contents()

                    for sNutrition2 in lNutritions2:
                        if 'カルシウム' in sNutrition2:
                            sCalcium = sNutrition2.split('カルシウム')[1].strip().replace('\n', '')
                        if 'マグネシウム' in sNutrition2:
                            sMagnesium = sNutrition2.split('マグネシウム')[1].strip().replace('\n', '')
                        if 'リン' in sNutrition2:
                            sPhosphorous = sNutrition2.split('リン')[1].strip().replace('\n', '')
                        if '鉄' in sNutrition2:
                            sIron = sNutrition2.split('鉄')[1].strip().replace('\n', '')
                        if '亜鉛' in sNutrition2:
                            sZinc = sNutrition2.split('亜鉛')[1].strip().replace('\n', '')
                        if 'ヨウ素' in sNutrition2:
                            sIodine = sNutrition2.split('ヨウ素')[1].strip().replace('\n', '')
                        if 'コレステロール' in sNutrition2:
                            sCholesterol = sNutrition2.split('コレステロール')[1].strip().replace('\n', '')
                        if 'ビタミンB1' in sNutrition2:
                            sVitamin_B1 = sNutrition2.split('ビタミンB1')[1].strip().replace('\n', '')
                        if 'ビタミンB2' in sNutrition2:
                            sVitamin_B2 = sNutrition2.split('ビタミンB2')[1].strip().replace('\n', '')
                        if 'ビタミンC' in sNutrition2:
                            sVitamin_C = sNutrition2.split('ビタミンC')[1].strip().replace('\n', '')

                    lNutritions3 = await oPage.locator('.c-nutrition-table__cell--3').all_text_contents()

                    for sNutrition3 in lNutritions3:
                        if 'ビタミンB6' in sNutrition3:
                            sVitamin_B6 = sNutrition3.split('ビタミンB6')[1].strip().replace('\n', '')
                        if 'ビタミンB12' in sNutrition3:
                            sVitamin_B12 = sNutrition3.split('ビタミンB12')[1].strip().replace('\n', '')
                        if '葉酸' in sNutrition3:
                            sFolate = sNutrition3.split('葉酸')[1].strip().replace('\n', '')
                        if 'ビタミンA' in sNutrition3:
                            sVitamin_A = sNutrition3.split('ビタミンA')[1].strip().replace('\n', '')
                        if 'ビタミンD' in sNutrition3:
                            sVitamin_D = sNutrition3.split('ビタミンD')[1].strip().replace('\n', '')
                        if 'ビタミンK' in sNutrition3:
                            sVitamin_K = sNutrition3.split('ビタミンK')[1].strip().replace('\n', '')
                        if 'ビタミンE' in sNutrition3:
                            sVitamin_E = sNutrition3.split('ビタミンE')[1].strip().replace('\n', '')
                        if '飽和脂肪酸' in sNutrition3:
                            sSaturated_fatty_acid = sNutrition3.split('飽和脂肪酸')[1].strip().replace('\n', '')
                        if '一価不飽和脂肪酸' in sNutrition3:
                            sMonounsaturated_fatty_acid = sNutrition3.split('一価不飽和脂肪酸')[1].strip().replace('\n', '')
                        if '多価不飽和脂肪酸' in sNutrition3:
                            sPolyunsaturated_fatty_acid = sNutrition3.split('多価不飽和脂肪酸')[1].strip().replace('\n', '')

                    # 食料
                    lIngredients = await oPage.locator('.p-recipe-ingredient-list__item').all_text_contents()
                    # for Ingredient in Ingredients:
                        # print(Ingredient)
                    lIngredients_clean = [sIngredient.strip().replace('\n', '') for sIngredient in lIngredients]

                    print('------作法步驟-----')

                    # 作法
                    lSteps = await oPage.locator('.p-recipe-step__item').all_text_contents()
                    # for Step in Steps:
                        # print(Step)
                    lSteps_clean = [sStep.strip().replace('\n', '') for sStep in lSteps]

                    # 定位到所有步骤的图片元素
                    Image_elements = oPage.locator('.p-recipe-step__item-image')

                    # 获取所有图片元素的 src 属性（即图片的 URL）
                    lImage_urls = await Image_elements.evaluate_all("elements => elements.map(e => e.getAttribute('src'))")

                    # 遍历图片 URL 列表，下载图片并转换为 Base64
                    lStep_images_base64 = []
                    for sImage_url in lImage_urls:
                        # 直接使用图片的 URL 下载图片并转换为 Base64
                        sImage_base64 = await fetch_image_as_base64(oPage, sImage_url)
                        if sImage_base64:
                            lStep_images_base64.append(sImage_base64)

                    dDish_data = {
                        'name': sDishname_clean,
                        'image': sDish_image_base64,
                        'likes': sLikes_count_clean,
                        'tags': ", ".join(lTags_clean),
                        'indications': ", ".join(lIndications_clean),
                        'Calories': sCalorie,
                        'Salt': sSalt,
                        'Protein': sProtein,
                        'Total_fat': sFat,
                        'Total_Carbohydrate': sCarbohydrate,
                        'Total_sugar': sSugar,
                        'Dietary_fiber': sDietary_fiber,
                        'Soluble_fiber': sSoluble_fiber,
                        'Insoluble_fiber': sInsoluble_fiber,
                        'K': sPotassium,
                        'Ca': sCalcium,
                        'Mg': sMagnesium,
                        'P': sPhosphorous,
                        'Fe': sIron,
                        'Zn': sZinc,
                        'I': sIodine,
                        'Cholesterol': sCholesterol,
                        'Vitamin_B1': sVitamin_B1,
                        'Vitamin_B2': sVitamin_B2,
                        'Vitamin_C': sVitamin_C,
                        'Vitamin_B6': sVitamin_B6,
                        'Vitamin_B12': sVitamin_B12,
                        'Folate': sFolate,
                        'Vitamin_A': sVitamin_A,
                        'Vitamin_D': sVitamin_D,
                        'Vitamin_K': sVitamin_K,
                        'Vitamin_E': sVitamin_E,
                        'Saturated_fatty_acid': sSaturated_fatty_acid,
                        'Monounsaturated_fatty_acid': sMonounsaturated_fatty_acid,
                        'Polyunsaturated_fatty_acid': sPolyunsaturated_fatty_acid,
                        'Ingredients': ", ".join(lIngredients_clean),
                        'Steps': lSteps_clean,
                        'Step_images_Base64': lStep_images_base64
                    }

                    await save_to_db(dDish_data)

                    break
                except Exception as e:  # 捕获可能发生的异常
                    print(f"遇到錯誤：{e}，嘗試返回並重試")
                    await oPage.go_back()  # 返回前一页
                    iRetry_count += 1  # 重试计数器加1
                    if iRetry_count >= 5:
                        print("重試次數超過，跳過當前連結")
                        break  # 跳出循环，处理下一个链接
            # 使用浏览器的后退功能返回列表页，这样不需要重新加载初始URL
            await oPage.go_back()
        bNext_page_button_exists = await oPage.is_visible('span.next a[rel="next"]')
        if bNext_page_button_exists:
            try:
                await oPage.click('span.next a[rel="next"]')
                await oPage.wait_for_load_state('networkidle')
            except Exception as e:
                print(f"訪問時錯誤：{e}，嘗試重新加載")
                # 如果“下一頁”不存在，break
                continue
        else:
            print("已達最後一頁")
            break
    # ---------------------
    await oContext.close()
    await oBrowser.close()


async def main() -> None:
    async with async_playwright() as playwright:
        await run(playwright)

# asyncio.run(main())
# async def simple_test():
#     print("Simple async test")
#
# asyncio.run(simple_test())