stephen.yu/RecipeCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

third commit

Browse Source
This commit is contained in:
stephen.yu
2024-03-15 10:32:44 +08:00
parent 50abf465c8
commit 172b67a716
1 changed files with 54 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
Recipe/views.py
View File

@@ -22,22 +22,22 @@ def sync_main():
async def save_to_db(dDish_data): async def save_to_db(dDish_data):
# 将异步ORM操作转换为同步，以适应Django ORM # 将异步ORM操作转换为同步，以适应Django ORM
dish, created = await sync_to_async(Dish.objects.update_or_create)( oDish, bCreated = await sync_to_async(Dish.objects.update_or_create)(
name= dDish_data['name'], name= dDish_data['name'],
defaults= dDish_data defaults= dDish_data
) )
sAction = "added" if created else "updated" sAction = "added" if bCreated else "updated"
print(f"Dish '{dish.name}' was {sAction}.") print(f"Dish '{oDish.name}' was {sAction}.")
# 將圖片轉為base64 # 將圖片轉為base64
async def fetch_image_as_base64(page, image_url): async def fetch_image_as_base64(oPage, sImage_url):
print("Fetching image from URL:", image_url) print("Fetching image from URL:", sImage_url)
try: try:
response = await page.request.get(image_url) oResponse = await oPage.request.get(sImage_url)
if response.ok: if oResponse.ok:
image_data = await response.body() bytImage_data = await oResponse.body()
sImage_base64 = base64.b64encode(image_data).decode() sImage_base64 = base64.b64encode(bytImage_data).decode()
return sImage_base64 return sImage_base64
except Exception as e: except Exception as e:
print(f"Error fetching image: {e}") print(f"Error fetching image: {e}")
@@ -45,27 +45,27 @@ async def fetch_image_as_base64(page, image_url):
async def run(playwright: Playwright): async def run(playwright: Playwright):
browser = await playwright.chromium.launch(headless=False) oBrowser = await playwright.chromium.launch(headless=False)
context = await browser.new_context() oContext = await oBrowser.new_context()
page = await context.new_page() oPage = await oContext.new_page()
await page.goto("https://oishi-kenko.com/recipes") await oPage.goto("https://oishi-kenko.com/recipes")
await page.get_by_role("link", name="ログイン").click() await oPage.get_by_role("link", name="ログイン").click()
await page.get_by_role("link", name="メールアドレス でログイン").click() await oPage.get_by_role("link", name="メールアドレス でログイン").click()
await page.locator("#secure_account_credential_email").click() await oPage.locator("#secure_account_credential_email").click()
await page.locator("#secure_account_credential_email").fill("asd851117005545@gmail.com") await oPage.locator("#secure_account_credential_email").fill("asd851117005545@gmail.com")
await page.locator("#secure_account_credential_email").press("Tab") await oPage.locator("#secure_account_credential_email").press("Tab")
await page.locator("#secure_account_credential_password").fill("a22897051") await oPage.locator("#secure_account_credential_password").fill("a22897051")
await page.get_by_role("button", name="ログイン").click() await oPage.get_by_role("button", name="ログイン").click()
await page.goto("https://oishi-kenko.com/recipes") await oPage.goto("https://oishi-kenko.com/recipes")
while True: while True:
# 訂位到所有匹配連結 # 訂位到所有匹配連結
links = page.locator('a.p-recipe-list-item__title-link') oLinks = oPage.locator('a.p-recipe-list-item__title-link')
# 獲取連結數量 # 獲取連結數量
iLink_count = await links.count() iLink_count = await oLinks.count()
# 點擊每個連結 # 點擊每個連結
for i in range(iLink_count): for i in range(iLink_count):
@@ -74,47 +74,47 @@ async def run(playwright: Playwright):
# 使用 nth(i) 定位第 i 個元素，並點擊 # 使用 nth(i) 定位第 i 個元素，並點擊
try: try:
await page.locator('a.p-recipe-list-item__title-link').nth(i).click() await oPage.locator('a.p-recipe-list-item__title-link').nth(i).click()
# 等待頁面 # 等待頁面
await page.wait_for_load_state('networkidle') await oPage.wait_for_load_state('networkidle')
# await asyncio.sleep(5) # await asyncio.sleep(5)
print("------菜名-----") print("------菜名-----")
# 輸出名稱 # 輸出名稱
sDishname = await page.text_content('.p-recipe-detail__title') sDishname = await oPage.text_content('.p-recipe-detail__title')
print(sDishname) print(sDishname)
sDishname_clean = sDishname.strip().replace('\n', '') sDishname_clean = sDishname.strip().replace('\n', '')
# 菜名圖片 # 菜名圖片
image_element = page.locator('.p-recipe-detail__photo-image--pc-only') oImage_element = oPage.locator('.p-recipe-detail__photo-image--pc-only')
# 从元素的 'src' 属性中获取图片的 URL # 从元素的 'src' 属性中获取图片的 URL
sImage_url = await image_element.get_attribute('src') sImage_url = await oImage_element.get_attribute('src')
# 确保获取到的 URL 不为空 # 确保获取到的 URL 不为空
if sImage_url: if sImage_url:
# 获取图片的 Base64 编码 # 获取图片的 Base64 编码
sDish_image_base64 = await fetch_image_as_base64(page, sImage_url) sDish_image_base64 = await fetch_image_as_base64(oPage, sImage_url)
else: else:
sDish_image_base64 = None sDish_image_base64 = None
# print("------按讚數-----") # print("------按讚數-----")
# 輸出按讚數 # 輸出按讚數
sLike_count = await page.text_content('.c-button-circle__top-text') sLike_count = await oPage.text_content('.c-button-circle__top-text')
# print(Likes_count) # print(Likes_count)
sLikes_count_clean = sLike_count.strip().replace('\n', '') sLikes_count_clean = sLike_count.strip().replace('\n', '')
# print("------標籤-----") # print("------標籤-----")
# 輸出標籤 # 輸出標籤
lTags = await page.locator('.c-button-round-tag__link').all_text_contents() lTags = await oPage.locator('.c-button-round-tag__link').all_text_contents()
# for tag in tags: # for tag in tags:
# print(tag) # print(tag)
lTags_clean = [sTag.strip() for sTag in lTags] lTags_clean = [sTag.strip() for sTag in lTags]
# print("------適應症-----") # print("------適應症-----")
# 輸出適應症 # 輸出適應症
lIndications = await page.locator('.c-recipes-relevant-dietary-concerns__text').all_text_contents() lIndications = await oPage.locator('.c-recipes-relevant-dietary-concerns__text').all_text_contents()
# for Indication in Indications: # for Indication in Indications:
# print(Indication) # print(Indication)
lIndications_clean = [sIndication.strip() for sIndication in lIndications] lIndications_clean = [sIndication.strip() for sIndication in lIndications]
@@ -154,7 +154,7 @@ async def run(playwright: Playwright):
sMonounsaturated_fatty_acid = '' sMonounsaturated_fatty_acid = ''
sPolyunsaturated_fatty_acid = '' sPolyunsaturated_fatty_acid = ''
lNutritions1 = await page.locator('.c-nutrition-table__cell--1').all_text_contents() lNutritions1 = await oPage.locator('.c-nutrition-table__cell--1').all_text_contents()
# for nutrition in nutritions1: # for nutrition in nutritions1:
# print(nutrition) # print(nutrition)
# nutritions_clean = [nutrition.strip().replace('\n', '') for nutrition in nutritions] # nutritions_clean = [nutrition.strip().replace('\n', '') for nutrition in nutritions]
@@ -180,7 +180,7 @@ async def run(playwright: Playwright):
if 'カリウム' in sNutrition1: if 'カリウム' in sNutrition1:
sPotassium = sNutrition1.split('カリウム')[1].strip().replace('\n', '') sPotassium = sNutrition1.split('カリウム')[1].strip().replace('\n', '')
lNutritions2 = await page.locator('.c-nutrition-table__cell--2').all_text_contents() lNutritions2 = await oPage.locator('.c-nutrition-table__cell--2').all_text_contents()
for sNutrition2 in lNutritions2: for sNutrition2 in lNutritions2:
if 'カルシウム' in sNutrition2: if 'カルシウム' in sNutrition2:
@@ -204,7 +204,7 @@ async def run(playwright: Playwright):
if 'ビタミンC' in sNutrition2: if 'ビタミンC' in sNutrition2:
sVitamin_C = sNutrition2.split('ビタミンC')[1].strip().replace('\n', '') sVitamin_C = sNutrition2.split('ビタミンC')[1].strip().replace('\n', '')
lNutritions3 = await page.locator('.c-nutrition-table__cell--3').all_text_contents() lNutritions3 = await oPage.locator('.c-nutrition-table__cell--3').all_text_contents()
for sNutrition3 in lNutritions3: for sNutrition3 in lNutritions3:
if 'ビタミンB6' in sNutrition3: if 'ビタミンB6' in sNutrition3:
@@ -229,7 +229,7 @@ async def run(playwright: Playwright):
sPolyunsaturated_fatty_acid = sNutrition3.split('多価不飽和脂肪酸')[1].strip().replace('\n', '') sPolyunsaturated_fatty_acid = sNutrition3.split('多価不飽和脂肪酸')[1].strip().replace('\n', '')
# 食料 # 食料
lIngredients = await page.locator('.p-recipe-ingredient-list__item').all_text_contents() lIngredients = await oPage.locator('.p-recipe-ingredient-list__item').all_text_contents()
# for Ingredient in Ingredients: # for Ingredient in Ingredients:
# print(Ingredient) # print(Ingredient)
lIngredients_clean = [sIngredient.strip().replace('\n', '') for sIngredient in lIngredients] lIngredients_clean = [sIngredient.strip().replace('\n', '') for sIngredient in lIngredients]
@@ -237,13 +237,13 @@ async def run(playwright: Playwright):
print('------作法步驟-----') print('------作法步驟-----')
# 作法 # 作法
lSteps = await page.locator('.p-recipe-step__item').all_text_contents() lSteps = await oPage.locator('.p-recipe-step__item').all_text_contents()
# for Step in Steps: # for Step in Steps:
# print(Step) # print(Step)
lSteps_clean = [sStep.strip().replace('\n', '') for sStep in lSteps] lSteps_clean = [sStep.strip().replace('\n', '') for sStep in lSteps]
# 定位到所有步骤的图片元素 # 定位到所有步骤的图片元素
Image_elements = page.locator('.p-recipe-step__item-image') Image_elements = oPage.locator('.p-recipe-step__item-image')
# 获取所有图片元素的 src 属性（即图片的 URL） # 获取所有图片元素的 src 属性（即图片的 URL）
lImage_urls = await Image_elements.evaluate_all("elements => elements.map(e => e.getAttribute('src'))") lImage_urls = await Image_elements.evaluate_all("elements => elements.map(e => e.getAttribute('src'))")
@@ -252,7 +252,7 @@ async def run(playwright: Playwright):
lStep_images_base64 = [] lStep_images_base64 = []
for sImage_url in lImage_urls: for sImage_url in lImage_urls:
# 直接使用图片的 URL 下载图片并转换为 Base64 # 直接使用图片的 URL 下载图片并转换为 Base64
sImage_base64 = await fetch_image_as_base64(page, sImage_url) sImage_base64 = await fetch_image_as_base64(oPage, sImage_url)
if sImage_base64: if sImage_base64:
lStep_images_base64.append(sImage_base64) lStep_images_base64.append(sImage_base64)
@@ -302,23 +302,28 @@ async def run(playwright: Playwright):
break break
except Exception as e: # 捕获可能发生的异常 except Exception as e: # 捕获可能发生的异常
print(f"遇到错误：{e}，尝试返回并重试") print(f"遇到错误：{e}，尝试返回并重试")
await page.go_back() # 返回前一页 await oPage.go_back() # 返回前一页
iRetry_count += 1 # 重试计数器加1 iRetry_count += 1 # 重试计数器加1
if iRetry_count >= 3: if iRetry_count >= 3:
print("重试次数超限，跳过当前链接") print("重试次数超限，跳过当前链接")
break # 跳出循环，处理下一个链接 break # 跳出循环，处理下一个链接
# 使用浏览器的后退功能返回列表页，这样不需要重新加载初始URL # 使用浏览器的后退功能返回列表页，这样不需要重新加载初始URL
await page.go_back() await oPage.go_back()
bNext_page_button_exists = await oPage.is_visible('span.next a[rel="next"]')
try: if bNext_page_button_exists:
await page.click('span.next a[rel="next"]') try:
await page.wait_for_load_state('networkidle') await oPage.click('span.next a[rel="next"]')
except Error: await oPage.wait_for_load_state('networkidle')
# 如果“下一頁”不存在，break except Exception as e:
print(f"訪問時錯誤：{e}，嘗試重新加載")
# 如果“下一頁”不存在，break
continue
else:
print("已達最後一頁")
break break
# --------------------- # ---------------------
await context.close() await oContext.close()
await browser.close() await oBrowser.close()
async def main() -> None: async def main() -> None: