最近回老家了一趟，爬取簡書的程序也就停下來了，回到長沙繼續(xù)爬，很高興的爬到了300W左右，導出來一看，好多重復的，我記得我說過設置過的，拿到代碼一看，暈：

插入的是author_infos表，但判斷卻是author_url表，然后我就打算去重后調用url爬取用戶詳細信息，可耐mongodb去重不會，自己百度了下也沒搞明白；再者，向右前輩說我爬取字段太少，那我就想重新修改爬取好了（已哭暈在廁所）。

代碼

import requests
from lxml import etree
import time
import pymongo

client = pymongo.MongoClient('localhost', 27017)
jianshu = client['jianshu']
author_infos = jianshu['author_infos']

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
    'Connection':'keep-alive'
}

def get_article_url(url,page):
    link_view = '{}?order_by=added_at&page={}'.format(url,str(page))
    try:
        html = requests.get(link_view,headers=headers)
        selector = etree.HTML(html.text)
        infos = selector.xpath('//div[@class="name"]')
        for info in infos:
            author_name = info.xpath('a/text()')[0]
            authorurl = info.xpath('a/@href')[0]
            if 'http://m.itdecent.cn'+ authorurl in [item['url'] for item in author_infos.find()]:
                pass
            else:
            # print('http://m.itdecent.cn'+authorurl,author_name)
            #     author_infos.insert_one({'author_name':author_name,'author_url':'http://m.itdecent.cn'+authorurl})
                get_all_info('http://m.itdecent.cn'+authorurl)
                get_reader_url(authorurl)
        time.sleep(2)
    except requests.exceptions.ConnectionError:
        pass

# get_article_url('http://m.itdecent.cn/c/bDHhpK',2)
def get_reader_url(url):
    link_views = ['http://m.itdecent.cn/users/{}/followers?page={}'.format(url.split('/')[-1],str(i)) for i in range(1,100)]
    for link_view in link_views:
        try:
            html = requests.get(link_view,headers=headers)
            selector = etree.HTML(html.text)
            infos = selector.xpath('//li/div[@class="info"]')
            for info in infos:
                author_name = info.xpath('a/text()')[0]
                authorurl = info.xpath('a/@href')[0]
                # print(author_name,authorurl)
                # author_infos.insert_one({'author_name': author_name, 'author_url': 'http://m.itdecent.cn' + authorurl})
                get_all_info('http://m.itdecent.cn' + authorurl)
        except requests.exceptions.ConnectionError:
            pass
# get_reader_url('http://m.itdecent.cn/u/7091a52ac9e5')

def get_all_info(url):
    html = requests.get(url,headers=headers)
    selector = etree.HTML(html.text)
    try:
        author_name = selector.xpath('//a[@class="name"]/text()')[0]
        author_focus = selector.xpath('//div[@class="info"]/ul/li[1]/div/p/text()')[0]
        author_fans = selector.xpath('//div[@class="info"]/ul/li[2]/div/p/text()')[0]
        author_article = selector.xpath('//div[@class="info"]/ul/li[3]/div/p/text()')[0]
        author_write_amount = selector.xpath('//div[@class="info"]/ul/li[4]/div/p/text()')[0]
        author_get_like = selector.xpath('//div[@class="info"]/ul/li[5]/div/p/text()')[0]
        author_intrus = selector.xpath('//div[1]/div/div[2]/div[2]/div/text()')
        author_intru = selector.xpath('//div[1]/div/div[2]/div[2]/div/text()') if len(author_intrus) != 0 else "無"
        if selector.xpath('//span[@class="author-tag"]'):
            author_title = '簽約作者'
        else:
            author_title = '普通作者'
        infos = {
            'url':url,
            'name':author_name,
            'focus':author_focus,
            'fans':author_fans,
            'article':author_article,
            'write_amount':author_write_amount,
            'get_like':author_get_like,
            'intru':author_intru,
            'title':author_title
        }
        author_infos.insert_one(infos)
    except IndexError:
        pass

今天就先這樣，主要是記錄自己的學習過程。
本文已在版權印備案，如需轉載請訪問版權印。58708803