바 이 두 카페 파충류(사례 연습:GET 요청)

#!/usr/bin/env python
# -*- coding: utf-8 -*-


import urllib.request
import urllib.parse
import ssl


get_url = 'http://tieba.baidu.com/f?kw=%s&ie=utf-8&pn=%d'

#         
ssl._create_default_https_context = ssl._create_unverified_context

headers = {
	# GET /f?ie=utf-8&kw=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD&fr=search HTTP/1.1
    'Host':	'tieba.baidu.com',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Referer': 'http://tieba.baidu.com/f?ie=utf-8&kw=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD&fr=search',
    #     ，     ，              ！！！
    # 'Accept-Encoding':'gzip, deflate',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cookie': 'TIEBA_USERTYPE=16f2e80db0d2749d6d8940be; TIEBAUID=cb23caae14130a0d384a57f1; BAIDUID=5825D3624FFD2FF79AD102CCE35CF40D:FG=1; PSTM=1532620217; BIDUPSID=4CA0F78CD45B4F46C5E80CFE8C9EB708; bdshare_firstime=1534381675497; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; H_PS_PSSID=26524_1434_21122_26350; delPer=0; PSINO=1; Hm_lvt_98b9d8c2fd6608d564bf2ac2ae642948=1540133767,1540133797,1540202629,1540954230; wise_device=0; Hm_lpvt_98b9d8c2fd6608d564bf2ac2ae642948=1540966604',
    'Connection':'keep-alive'
}


def load_url_data(search_keywords, num):
    """
      :    ，    
    :param search_keywords:        ，
    :param num:        ，
    :return: response
    """

    for i in range(num):
        full_get_url = get_url % (search_keywords, i*50)
        headers_request = urllib.request.Request(url=full_get_url, headers=headers)
        response = urllib.request.urlopen(headers_request)

        # read()       ，   ，     
        #    ：
        # content = response.read().decode("utf-8") #   ！
        # UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte
        #   ： headers    'Accept-Encoding':'gzip, deflate',      。
        #    ：
        #   'Accept-Encoding':'gzip, deflate',   ，          ，    ！！！
        # content = response.read().decode("utf-8", errors="replace")

        content = response.read().decode("utf-8")
        write_to_file(content, i)


def write_to_file(content, i):
    """
      :           
    :param content:           ，
    :param i:   
    :return: html  
    """

    # open      ，           ，    ，         ，  mode='wb'，      encoding
    with open("./   %d .html" % (i+1), mode='wb') as fp:
        fp.write(content.encode('utf-8'))
        print("   %d     " % (i+1))


if __name__ == '__main__':

    search_keywords = input("          ：")

    #    ：  
    # search_keywords = urllib.parse.urlencode(search_keywords).encode('utf-8')
    # TypeError: not a valid non-string sequence or mapping object
    #   ：           ， urllib.parse.urlencode().encode() | urlencode()  post_url

    #    ：OK
    #               urllib.parse.quote() | quote()|unquote()   get_url
    search_keywords = urllib.parse.quote(search_keywords)

    try:
        num = int(input("         ："))
    except Exception as e:
        print("        ：")
        num = int(input("         ："))

    load_url_data(search_keywords, num)