Pyspider 프레임워크의 대중 평가 데이터 캡처

수요

코드

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-08-29 14:37:07
# Project: dianping_vi

from pyspider.libs.base_handler import *
import datetime
import re
import json
import copy

from pymongo import MongoClient

#  
DB_IP = 
DB_PORT = 

#DB_IP = '127.0.0.1'
#DB_PORT = 27017

client = MongoClient(host=DB_IP, port=DB_PORT)

# admin  ， - - 
db_auth = client.admin
db_auth.authenticate( )

DB_NAME = 'research'
DB_COL = 'dianping'
db = client[DB_NAME]
col = db[DB_COL]



detail_headers = {
    'Host': 'www.dianping.com',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36',
    'Cookie': 'hc.v=c433e5ea-ff94-9d82-2544-871b013c64eb.1536116212; _lxsdk_cuid=165a7a93ffcc8-0885d455e400d4-3b7b0d58-1aeaa0-165a7a93ffcc8; _lxsdk=165a7a93ffcc8-0885d455e400d4-3b7b0d58-1aeaa0-165a7a93ffcc8; _lxsdk_s=165a8c7b269-b24-ec-e63%7C%7C135',
    #'Cookie': '_lxsdk_cuid=165419b77c0c8-0b7bab6ed7c246-1e2e130c-1fa400-165419b77c1c8; _lxsdk=165419b77c0c8-0b7bab6ed7c246-1e2e130c-1fa400-165419b77c1c8; _hc.v=b53c090b-d406-9c02-4cf2-ef330bf04f87.1534404033; switchcityflashtoast=1; source=m_browser_test_33; pvhistory="6L+U5ZuePjo8L3N1Z2dlc3QvZ2V0SnNvbkRhdGE/Y2FsbGJhY2s9anNvbnBfMTUzNDQwNDI0NjYxOV82NTg3NT46PDE1MzQ0MDQyNDY2NzddX1s="; m_flash2=1; default_ab=citylist%3AA%3A1%7Cshop%3AA%3A1%7Cindex%3AA%3A1%7CshopList%3AA%3A1; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; _lxsdk_s=%7C%7C0',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',
     'Accept-Language': 'en-US,en;q=0.9',
     'Cache-Control': 'no-cache',
     'Upgrade-Insecure-Requests': '1',
     'Connection': 'keep-alive',
     'Pragma': 'no-cache',
}

def parse_score(taste, doc):
    if len(taste) == 2:
        taste_score = [num_map.get(x, '0') for x in taste]
        taste = float('.'.join(taste_score))
    elif len(taste) == 1:
        taste_score = num_map.get(taste[0], '0')
        _flag = doc.xpath('//span[@id="comment_score"]/span[1]/text()')[1]
        if _flag.startswith('1'):
            taste = float('1.' + taste_score)
        else:
            taste = float(taste_score + '1.')
    else:
        taste = 0
    return taste



def get_today():
    return datetime.datetime.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d')


class Handler(BaseHandler):
    crawl_config = {
        'proxy': '',
        "headers": {
            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'
        },
        'retries': 6
        
    }

    @every(minutes=24 * 60)
    def on_start(self):
        url = 'https://m.dianping.com/citylist'
        
        self.crawl(url, callback=self.get_city_id)
        
        
    @config(age=60)
    def get_city_id(self, response):
        #  url
        url = 'https://m.dianping.com/{}/ch10/d1?from=m_nav_1_meishi'
        
        result = re.findall('window.PAGE_INITIAL_STATE = ({.*})', response.text)[0]
        print(result)
        
        city_data = json.loads(result)
        for each in city_data['hotcity']['data']['hotCity']:
            item = {
                'city_name': each['cityName'],
                'city_id': each['cityId'],
                'city_pyname': each['cityEnName'],
            }
            city_pyname = each['cityEnName']

            print(each['cityName'])

            self.crawl(url.format(city_pyname), validate_cert=False, callback=self.get_area_food, save=item)

        for each in city_data['list']['data']['cityData']:
            for i in each['list']:
                item = {
                    'city_name': i['cityName'],
                    'city_id': i['cityId'],
                    'city_pyname': i['cityEnName'],
                }
                city_pyname = i['cityEnName']

                print(i['cityName'], '-----',city_pyname)
                #if city_pyname == 'rongcang':
                self.crawl(url.format(city_pyname), validate_cert=False, callback=self.get_area_food, save=item)

                
    @config(age=60)
    def get_area_food(self, response):
        result = re.findall('window.PAGE_INITIAL_STATE = ({.*})', response.text)[0]
        item = response.save
        print(item)
        city_pyname = item.pop('city_pyname')

        data = json.loads(result)
        areas = data['mapiSearch']['data']['regionNavs']
        areas = [x for x in areas if x['parentId'] == 0 and x['count'] > 0]
        # {u'count': 9828, u'name': u'\u897f\u57ce\u533a', u'regionType': 0, u'parentId': 0, u'lat': 0, u'lng': 0, u'id': 16}

        foods = data['mapiSearch']['data']['categoryNavs']
        foods = [x for x in foods if x['parentId'] == 10 and x["id"] != 10]
        

        city_id = item.pop('city_id')
        for area in areas:
            for food in foods:
                _item = copy.deepcopy(item)
                _item['region'] = area['name']
                _item['category'] = food['name']
                category_id = food['id']
                region_id = area['id']
                print(area['name'], food['name'], category_id, region_id)
                _url = 'http://www.dianping.com/{}/ch10/g{}r{}p1'.format(city_pyname, category_id, region_id)
                print(_url)
                self.crawl(_url, callback=self.get_next_page, save={'item': item, 'url': _url})


    @config(age=60)
    def get_next_page(self, response):
        _item = response.save['item']
        doc = response.etree
        
        ##  ------- if ， 
        if response.save.get('url'):
            url = response.save['url']
            pages = doc.xpath('//a[@class="PageLink"]/text()')
            if len(pages) > 0 or pages[-1] != '1':
                max_page = int(pages[-1])
                for each in range(2, max_page + 1, 1):
                    _url = url.replace('p1', 'p{}'.format(each))
                    self.crawl(_url, callback=self.get_next_page, save={'item': _item})
                
        shops = doc.xpath('//div[@id="shop-all-list"]/ul/li')
        for shop in shops:
            item = copy.deepcopy(_item)
            shop_id = shop.xpath('.//div[@class="tit"]/a[1]/@data-shopid')[0]
            item['shop_id'] = shop_id
            
            name = shop.xpath('.//div[@class="tit"]/a[1]/@title')[0]
            item['name'] = name  #  
            
            is_ad = shop.xpath('.//div[@class="tit"]/a[2]/text()')
            if len(is_ad) == 0 or is_ad[0] != ' ':
                item['ad_shop'] = False  #  
            else:
                item['ad_shop'] = True  #  
            
            shoop_id = shop.xpath('.//div[@class="tit"]/a[1]/@data-shopid')[0]
            item['shop_id'] = shoop_id  #  id                 
            
            addr = shop.xpath('.//span[@class="addr"]/text()')
            item['addr'] = addr[0] if len(addr) > 0 else ''  # address
            
            region_tag = shop.xpath('.//div[@class="tag-addr"]/a[2]/span[@class="tag"]/text()')

            item['area'] = region_tag[0] if len(region_tag) > 0 else ''  #  
            
            review = shop.xpath('.//div[@class="comment"]/a[1]/b/text()')
            item['review_count'] = int(review[0]) if len(review) > 0  else 0  #  
            
            price = shop.xpath('.//div[@class="comment"]/a[2]/b/text()')
            item['price_text'] = int(price[0].replace('￥', '')) if len(price) > 0 else 0
            
            scores = shop.xpath('.//span[@class="comment-list"]/span/b/text()')
            if len(scores) == 3:
                taste, surrounding, service = scores
                item['taste'] = float(taste)
                item['surrounding'] = float(surrounding)
                item['service'] = float(service)
            else:
                item['taste'] = 0
                item['surrounding'] = 0
                item['service'] = 0
                
            star = shop.xpath('.//div[@class="comment"]/span/@class')[0]
            item['star'] = float(star.replace('sml-rank-stars sml-str', ''))/10
            
            ##  
            print(item)

            _url = "http://www.dianping.com/shop/{}/review_all".format(item['shop_id'])

            self.crawl(_url, callback=self.get_detail_page, save={'item': item}, headers=detail_headers)
            
              
            

    @config(age=60)
    def get_detail_page(self, response):
       
        item = response.save['item']
        doc = response.etree

        tags_list = doc.xpath("//div[@class='reviews-tags']/div[@class='content']/span")
        
        review_tags = []
        for each in tags_list:
            review_tags.append(''.join([i.strip() for i in each.xpath("./a/text()")[0].split('
')]))
           
        item["review_tags"] = review_tags    
        print(review_tags)
        
        #comments_list = doc.xpath("//div[@id='summaryfilter-wrapper']/div[class='comment-filter-box clearfix J-filter']/label")
        
        if doc.xpath("//label[@class='filter-item filter-pic']"):
            item["pic"] = u' ' + doc.xpath("//label[@class='filter-item filter-pic']/span[@class='count']/text()")[0]
        else:
            item["pic"] = ''
        
        print(item["pic"])
        if doc.xpath("//label[@class='filter-item filter-good']"):
            item["good"] = u' ' + doc.xpath("//label[@class='filter-item filter-good']/span[@class='count']/text()")[0]
        else:
            item["good"] = ''
          
        print(item["good"])
        if doc.xpath("//label[@class='filter-item filter-middle']"):
            item["middle"] = u' ' + doc.xpath("//label[@class='filter-item filter-middle']/span[@class='count']/text()")[0]
        else:
            item["middle"] = ''
        
        print(item["middle"])
        if doc.xpath("//label[@class='filter-item filter-bad']"):
            item["bad"] = u' ' + doc.xpath("//label[@class='filter-item filter-bad']/span[@class='count']/text()")[0]
        else:
            item["bad"] = ''
        
        print(item["bad"])
        
       
        item["date"] = get_today()
        item["update_time"] = datetime.datetime.now()

        print(item)

        yield item

 
                
    def on_result(self, result):
        super(Handler, self).on_result(result)
        if not result:
            return

        update_key = {
            'name': result['name'],
            'shop_id': result['shop_id'],
            'date': result["date"]
        }

        col.update(update_key, {'$set': result}, upsert=True)