프로 세 스 탱크 에서 프 록 시 와 검 측 프 록 시 가 져 오기

import requests
from lxml import etree
import multiprocessing

#      
def get_all_proxy(queue):
    url = 'http://www.xicidaili.com/nn/1'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
    }
    response = requests.get(url,headers=headers)

    html = response.text
    html = etree.HTML(html)

    ip_list = html.xpath('//table[@id="ip_list"]/tr/td[2]/text()')
    d_list = html.xpath('//table[@id="ip_list"]/tr/td[3]/text()')
    # print(ip_list,d_list)

    for i in range(0,len(ip_list)):
        proxy = 'http://' + ip_list[i] + ':' + d_list[i]
        # print(proxy)

        # print(porxy_list)
        queue.put(proxy)
# with open('xicidaili.html','wb') as f:
#     f.write(response.content)

#      
def check_all_proxy(proxy):
        # print(proxy)
    try:
        url = 'http://fanyi.baidu.com/sug'
        porxy_dict = {
            'http':proxy
        }
        try:
            response = requests.get(url,proxies=porxy_dict,timeout=3)
            if response.status_code == 200:
                print(proxy)
                # with open('ip.txt', 'a') as f:
                #     f.write(proxy+'
')
                return proxy

            else:
                print('   '*20)
                return proxy
        except:
            return None
    except Exception as e:
        print(e)

#   
if __name__ == '__main__':
    import time
    start_time = time.time()
    #     
    q = multiprocessing.Queue()
    # pool     ,          queue
    #       
    p = multiprocessing.Process(target=get_all_proxy, args=(q,))
    #   
    p.start()
    # proxy_list = get_all_proxy()
    #         

    pool = multiprocessing.Pool(50)
    result_list = []
    while True:
        try:
            proxy_str = q.get(timeout=5)
            # print('       ', proxy_str)
        except:
            break
        # print('apply_async   ')
        # proxy_and_queue = [proxy_str, result_q]
        proxy_res = pool.apply_async(check_all_proxy, (proxy_str,))
        result_list.append(proxy_res)
    # valid_proxy_list = check_all_proxy(proxy_list)

    valid_proxy_list = []
    for proxy_res in result_list:
        result = proxy_res.get()
        if result is None:
            pass
        else:
            valid_proxy_list.append(result)
            # print(result)

    print('All proxy we can get:')
    print(valid_proxy_list)
    pool.close()
    pool.join()
    p.join()