간단한 async/aiohttp 기반 파충류

import asyncio
import aiohttp
from lxml import etree
import queue

urlQ = queue.Queue()
f = open("title22.txt", "w",encoding='utf-8')

async def get_html(url):
    ck = """Hm_lvt_dbc355aef238b6c32b43eacbbf161c3c=1507966069,1509850072,1509851337,1509851651; Hm_lpvt_dbc355aef238b6c32b43eacbbf161c3c=1509851653"""
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36",
        "Referer": url,
        "Cookie": ck,
        "Host": "www.mzitu.com"
    }
    async with asyncio.Semaphore(5): #  5 
        async with aiohttp.ClientSession() as session:
            async with session.get(url, headers=headers) as html:
                # print(resp.status)
                response = await html.text(encoding="utf-8")
                # print(response)
                return response

async def parse():
    while True: #  while true
        if urlQ.empty():
            break
        url = urlQ.get()
        html = await get_html(url)
        selector = etree.HTML(html)
        titles = selector.xpath("//div[@class='postlist']//li/a/img/@alt")
        for title in titles:
            f.write(title + '
')
asyncio.ensure_future(parse())


urls = ["http://www.mzitu.com/page/{}/".format(i) for i in range(1, 157)]
for url in urls:
    urlQ.put(url)
loop = asyncio.get_event_loop()
tasks = [parse() for _ in range(50)] #  ， parse while true ， 50 ， 50 ， 
loop.run_until_complete(asyncio.wait(tasks))
loop.close()

f.close()

몇 개의 협동 파충류와 관련된 링크를 첨부합니다. 이 안에 요청에 대한 봉인이 있습니다. ua, 쿠키 같은 비동기 파충류: async/await와aiohttp의 사용, 그리고 예시Python에서 비동기 협동의 사용 방법을 소개합니다.