탭 전체 네트워크 데이터 - 아날로그 브라우저

1단계: 탭 첫 페이지 정보 얻기

 1 #         
 2 
 3 url = 'https://www.lagou.com/'
 4 headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36',
 5           'Connection': 'keep-alive',
 6            'Cookie': '_ga=GA1.2.759756292.1572002144; _gid=GA1.2.1024774356.1572002144; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1572002144; user_trace_token=20191025191854-3af2cb8b-f719-11e9-a08c-525400f775ce; PRE_UTM=; PRE_HOST=cn.bing.com; PRE_SITE=https%3A%2F%2Fcn.bing.com%2F; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; LGUID=20191025191854-3af2ce76-f719-11e9-a08c-525400f775ce; LGSID=20191025191854-3af2ccf6-f719-11e9-a08c-525400f775ce; index_location_city=%E5%85%A8%E5%9B%BD; _gat=1; LGRID=20191025192516-1eb14395-f71a-11e9-a607-5254005c3644; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1572002526; X_HTTP_TOKEN=7f3a7ebf1bb8dfc4987200275129b8d1ec41735636'
 7           }
 8 
 9 response = requests.get(url, headers=headers)
10 soup = BeautifulSoup(response.text, 'lxml')
11 # soup

2단계: 첫 페이지의 모든 직위 링크 얻기

#           

all_info = soup.select('div.mainNavs a')
position_urls = []
for i in all_info:
    url = i.get('href')
    position_urls.append(url)
# position_urls

3단계: 해당 링크의 직위 모집 정보 얻기

#            

position_info = []

for i in position_urls:
#     print(i)
    response_i = requests.get(i, headers = headers)
    soup = BeautifulSoup(response_i.text, 'lxml')
#     print(soup)
    con_list = soup.select('ul.item_con_list')[1].select('li')
#     print(con_list[0])

   #       
    for li in con_list:
        dic = {}
        dic['position_name'] = li.select('h3')[0].text
        dic['address'] = li.select('span.add')[0].text
        dic['salary&experience'] = li.select('div.li_b_l')[0].text.split('
')[1]
        dic['experience'] = li.select('div.li_b_l')[0].text.replace(' ','').split('
')[2].split('/')[0]
        dic['degree'] = li.select('div.li_b_l')[0].text.replace(' ','').split('
')[2].split('/')[1]
        dic['job_kind'] = li.select('div.li_b_l')[1].text.replace('
','/')
        position_info.append(dic)
print(position_info)