這是一個異步爬蟲示例。
創建自己爬蟲類，繼承Crawler類，重寫parse方法，調用run方法即進行爬取。
from urllib.parse import urlparse
import asyncio
import aiohttp
def save(content: any, filename: str, mode='a', encoding='utf-8', end='\n'):
with open(filename, mode=mode, encoding=encoding) as file:
if type(content) == str:
file.write(content + end)
else:
file.write(content)
print(f'The file is saved "{
filename}" successfully!')
class Crawler(object):
""" 爬蟲基類，所有爬蟲都應該繼承此類 """
def __init__(self, start_url: list[str]) -> None:
""" 初始化 :param start_url: 爬蟲起始列表 :param start_url: 爬蟲起始列表 :param domain: 域名 """
self.items = None
self.name = 'myspider'
self.start_url = start_url
self.domain = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(self.start_url[0]))
async def parse(self, response):
""" 從response中解析出所有目錄的URL鏈接 """
raise NotImplementedError
async def request(self, url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
response = await self.parse(await resp.text())
url_object = urlparse(url)
return dict(response=response, url=url) # 有返回值
async def run(self):
tasks = [asyncio.ensure_future(self.request(i)) for i in self.start_url]
await asyncio.wait(tasks)
results = await asyncio.gather(*tasks) # 獲取返回值
self.handle(results)
print("任務完成...")
def handle(self, items):
for item in items:
save(item.text(), "test.txt")
raise NotImplementedError
