第二周實(shí)戰(zhàn)作業(yè)：爬取10萬(wàn)商品數(shù)據(jù)

實(shí)戰(zhàn)計(jì)劃0430-石頭的練習(xí)作業(yè)

作業(yè)要求

實(shí)現(xiàn)效果

實(shí)現(xiàn)代碼

__author__ = 'daijielei'

'''
    獲取趕集網(wǎng)的二手物品分類信息，提取出各個(gè)分類的鏈接，并保存在一個(gè)數(shù)組中以備提取用
'''
from bs4 import BeautifulSoup
import requests

hosturl = 'http://bj.ganji.com'

def getGanjiAlltype(url='http://bj.ganji.com/wu/#all_category'):
    webdata = requests.get(url)
    soup = BeautifulSoup(webdata.text,'lxml')

    for item in soup.select('div.content div.clearfix dl.fenlei a'):
        print(hosturl + str(item.get('href')))
        #print('---------------------------------------------')


#getGanjiAlltype()

links = '''
    http://bj.ganji.com/jiaju/
    http://bj.ganji.com/rirongbaihuo/
    http://bj.ganji.com/chuangdian/
    http://bj.ganji.com/guizi/
    http://bj.ganji.com/zhuoyi/
    http://bj.ganji.com/shafachaji/
    http://bj.ganji.com/zixingchemaimai/
    http://bj.ganji.com/diandongche/
    http://bj.ganji.com/motuoche/
    http://bj.ganji.com/shouji/
    http://bj.ganji.com/shoujihaoma/
    http://bj.ganji.com/iphone/
    http://bj.ganji.com/nokia/
    http://bj.ganji.com/htc/
    http://bj.ganji.com/sanxingshouji/
    http://bj.ganji.com/motorola/
    http://bj.ganji.com/shouji/_%E5%B0%8F%E7%B1%B3/
    http://bj.ganji.com/shouji/_%E9%AD%85%E6%97%8F/
    http://bj.ganji.com/tongxuntaocan/
    http://bj.ganji.com/qqhao/
    http://bj.ganji.com/bangong/
    http://bj.ganji.com/nongyongpin/
    http://bj.ganji.com/bangongjiaju/
    http://bj.ganji.com/jiguangyitiji/
    http://bj.ganji.com/dayinji/z1/
    http://bj.ganji.com/shipinjiagongshebei/
    http://bj.ganji.com/shengchanjiamengshebei/
    http://bj.ganji.com/jichuang/
    http://bj.ganji.com/tuolaji/
    http://bj.ganji.com/jiadian/
    http://bj.ganji.com/dianshi/
    http://bj.ganji.com/bingxiang/
    http://bj.ganji.com/kongtiao/
    http://bj.ganji.com/reshuiqi/
    http://bj.ganji.com/xiyiji/
    http://bj.ganji.com/diancilu/
    http://bj.ganji.com/weibolu/
    http://bj.ganji.com/yueqiyinxiang/
    http://bj.ganji.com/ershoubijibendiannao/
    http://bj.ganji.com/pingbandiannao/z1/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E8%8B%B9%E6%9E%9C/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E8%81%94%E6%83%B3/
    http://bj.ganji.com/ershoubijibendiannao/z1/_Thinkpad/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E7%B4%A2%E5%B0%BC/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E6%88%B4%E5%B0%94/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E5%8D%8E%E7%A1%95/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E6%83%A0%E6%99%AE/
    http://bj.ganji.com/ruanjiantushu/
    http://bj.ganji.com/yueqi/
    http://bj.ganji.com/yinxiang/
    http://bj.ganji.com/yundongqicai/
    http://bj.ganji.com/yingyouyunfu/
    http://bj.ganji.com/tongche/
    http://bj.ganji.com/tongzhuang/
    http://bj.ganji.com/yingerche/
    http://bj.ganji.com/yingerchuang/z1/
    http://bj.ganji.com/niaobushi/
    http://bj.ganji.com/wanju/
    http://bj.ganji.com/naifen/
    http://bj.ganji.com/diannao/
    http://bj.ganji.com/taishidiannaozhengji/
    http://bj.ganji.com/xianka/
    http://bj.ganji.com/cpu/
    http://bj.ganji.com/yingpan/
    http://bj.ganji.com/luyouqi/
    http://bj.ganji.com/3gwangka/
    http://bj.ganji.com/xianzhilipin/
    http://bj.ganji.com/shoucangpin/
    http://bj.ganji.com/qitalipinzhuanrang/
    http://bj.ganji.com/baojianpin/
    http://bj.ganji.com/xiaofeika/
    http://bj.ganji.com/fushixiaobaxuemao/
    http://bj.ganji.com/meironghuazhuang/
    http://bj.ganji.com/fushi/
    http://bj.ganji.com/xiangbao/
    http://bj.ganji.com/xuemao/
    http://bj.ganji.com/shoubiao/
    http://bj.ganji.com/shipin/
    http://bj.ganji.com/huazhuangpin/
    http://bj.ganji.com/hufupin/
    http://bj.ganji.com/shuma/
    http://bj.ganji.com/shumaxiangji/
    http://bj.ganji.com/shumashexiangji/
    http://bj.ganji.com/youxiji/
    http://bj.ganji.com/suishenting/
    http://bj.ganji.com/yidongcunchu/
    http://bj.ganji.com/laonianyongpin/
    http://bj.ganji.com/zibubaojian/z2/
    http://bj.ganji.com/anmobaojian/z1/
    http://bj.ganji.com/bawanwujian/
    http://bj.ganji.com/xuniwupin/
    http://bj.ganji.com/qitawupin/
    http://bj.ganji.com/ershoufree/
    http://bj.ganji.com/wupinjiaohuan/
    http://bj.ganji.com/zhuanqu_anjia/all/
    http://bj.ganji.com/zhuanqu_jiaren/all/
    http://bj.ganji.com/zhuanqu_shenghuo/all/
'''

__author__ = 'daijielei'

'''
    爬蟲，抓取趕集網(wǎng)下的相關(guān)信息

'''
from bs4 import BeautifulSoup
import requests
import pymongo
import time
import random
import IP_Proxy
import datetime


__Test__ = ''   # 'test'   #'該開(kāi)關(guān)用來(lái)做測(cè)試用的控制'，打開(kāi)時(shí)，不會(huì)寫入數(shù)據(jù)庫(kù)和做數(shù)據(jù)庫(kù)的判斷
__Prosey__ = '' # 'use'  #該開(kāi)關(guān)用來(lái)測(cè)試控制是否使用IP代理
'''
-----------------------------------------------------------------------------------------------------
|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
以下為數(shù)據(jù)庫(kù)相關(guān)代碼
更多的mongoDB可以查看如下：http://www.runoob.com/mongodb/mongodb-indexing.html
                        https://blog.codecentric.de/files/2012/12/MongoDB-CheatSheet-v1_0.pdf
1、mongoDB可以通過(guò)建立索引來(lái)提高查詢效率
2、可以使用條件操作符來(lái)進(jìn)行> >= < <= !=的查詢
3、可以使用$type來(lái)查詢指定同一個(gè)名稱下指定類型的數(shù)據(jù)
4、可以使用limit來(lái)限制查詢的結(jié)果的數(shù)量，使用skip來(lái)跳過(guò)指定數(shù)目的結(jié)果
5、可以進(jìn)行排序、索引、聚合等



'''
client = pymongo.MongoClient('localhost',27017)
ganjiDB = client['ganjiDB']


'''
    ---------            gangjiGetlistChecksheet該表用來(lái)保存已經(jīng)爬過(guò)的列表，防止重新爬               --------
    ---------            gangjiGetInfoChecksheet該表用來(lái)保存已經(jīng)爬過(guò)的商品信息，防止重新爬               --------
'''
gangjiGetlistChecksheet = ganjiDB['gangjiGetlistChecksheet']#保存爬過(guò)的list,如果已經(jīng)爬過(guò)就不再爬了
gangjiGetInfoChecksheet = ganjiDB['gangjiGetInfoChecksheet']#保存爬過(guò)的信息,如果已經(jīng)爬過(guò)就不再爬了
#將爬過(guò)的列表的URL進(jìn)行保存
def insertgangjiGetlistChecksheet(data):
    if(__Test__ == 'test'):return
    gangjiGetlistChecksheet.insert_one(data)
#將爬過(guò)的商品的URL進(jìn)行保存
def insertgangjiGetInfoChecksheet(data):
    if(__Test__ == 'test'):return
    gangjiGetInfoChecksheet.insert_one(data)
#判斷該列表URL是否已經(jīng)爬過(guò)
def ifexistgangjiGetlistChecksheet(url):
    if(__Test__ == 'test'):return False
    ifexist = False
    if(gangjiGetlistChecksheet.find({'url':url}).count()>0):
        ifexist = True
    return ifexist
#判斷該商品URL是否已經(jīng)爬過(guò)
def ifexistgangjiGetInfoChecksheet(url):
    if(__Test__ == 'test'):return False
    ifexist = False
    if(gangjiGetInfoChecksheet.find({'url':url}).count()>0):
        ifexist = True
    return ifexist
'''
    ---------            ganjiUrlSheet該表用來(lái)保存已經(jīng)爬過(guò)的商品URL，以用判斷用               --------
'''
ganjiUrlSheet = ganjiDB['ganjiUrlSheet']#保存爬下來(lái)的趕集商品url內(nèi)容
#對(duì)商品的url表進(jìn)行插入
def insertganjiUrlSheet(data={}):
    if(__Test__ == 'test'):return
    ganjiUrlSheet.insert_one(data)
#對(duì)商品的url表進(jìn)行顯示
def showganjiUrlSheet():
    for item in ganjiUrlSheet.find():
        print(item)
#對(duì)商品的url表進(jìn)行刪除
def deletAllganjiUrlSheet():
    ganjiUrlSheet.delete_many({})
    print(ganjiUrlSheet.count())
#對(duì)商品的url表是否存在進(jìn)行判斷
def ifexistganjiUrlSheet(url):
    if(__Test__ == 'test'):return False
    ifexist = False
    if ganjiUrlSheet.find({'url':url}).count() > 0:
        ifexist = True
    return ifexist



'''
    ---------            ganjiInfoSheet該表用來(lái)保存已經(jīng)爬過(guò)的商品信息，以做判斷用               --------
'''
ganjiInfoSheet = ganjiDB['ganjiInfoSheet']
#對(duì)商品信息插入對(duì)應(yīng)的表
def insertganjiInfoSheet(data={}):
    if(__Test__ == 'test'):return
    ganjiInfoSheet.insert_one(data)
#顯示全部商品信息
def showganjiInfoSheet():
    for item in ganjiInfoSheet.find():
        print(item)


'''
    ---------            對(duì)多個(gè)表進(jìn)行操作               --------
'''
#顯示當(dāng)前表中已有的數(shù)據(jù)
def showAllDBcount():
    print('趕集url表中以存數(shù)據(jù)：'+str(ganjiUrlSheet.count()))
    print('趕集商品表中以存數(shù)據(jù)：'+str(ganjiInfoSheet.count()))

#showAllDBcount()

#清除全部表中的全部數(shù)據(jù)
def clearAllganjisheet():
    ganjiUrlSheet.delete_many({})
    gangjiGetlistChecksheet.delete_many({})
    gangjiGetInfoChecksheet.delete_many({})
    ganjiInfoSheet.delete_many({})
    print(ganjiUrlSheet.count())
    print(gangjiGetlistChecksheet.count())
    print(gangjiGetInfoChecksheet.count())
    print(ganjiInfoSheet.count())

'''
-----------------------------------------------------------------------------------------------------
|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
以下為爬蟲部分的代碼

gangji_getlist  用來(lái)抓取列表里的商品URL等信息

gangji_getInfo  用來(lái)抓取商品的相關(guān)信息

'''

headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
    'Connection':'keep-alive',
}
#代理服務(wù)器可以從中選擇：http://cn-proxy.com/
#獲取隨機(jī)的代理服務(wù)器，以緩解被服務(wù)器禁用的風(fēng)險(xiǎn)
ip_proxy = ['1','2']#IP_Proxy.getProxy() 不用Proxy
if __Prosey__ == 'use':
    ip_proxy = IP_Proxy.getProxy()
def getRandomProxy():
    proxies = {
        'http':random.choice(ip_proxy),
    }
    return  proxies

def delProxy(proxy):
    proxy = proxy['http']
    index = list(ip_proxy).index(proxy)
    del ip_proxy[index]
    print('清除無(wú)效的ip_proxy,剩余數(shù)量：'+str(len(ip_proxy)))


#connect()用來(lái)連接URL獲取頁(yè)面的html數(shù)據(jù)
def connect(url):
    try:
        proxies = getRandomProxy()
        if __Prosey__ == 'use':
            webdata = requests.get(url,headers=headers,proxies=proxies) #使用proxies,來(lái)緩解被阻攔的風(fēng)險(xiǎn)
        else:
            webdata = requests.get(url)
        return webdata
    except (requests.exceptions.ConnectionError,TimeoutError):
        delProxy(proxies)
        print('ConnectionError')
    return None

#connectTry()因?yàn)樵擁?yè)面鏈接經(jīng)常失效，故通過(guò)一個(gè)重連機(jī)制來(lái)不斷重試
def connectTry(url):
    count = 0
    webdata = None
    while not webdata:  #如果沒(méi)有獲取到數(shù)據(jù)，繼續(xù)重試
        time.sleep(1)
        webdata = connect(url)
        count = count + 1
        print('第'+str(count)+'次重試')
        if(count > 100):    #超過(guò)最大重試次數(shù)，直接退出
            return None
    return webdata



'''                              gangji_getlist  用來(lái)抓取列表里的商品URL等信息                            '''
def gangji_getlist(url='http://bj.ganji.com/motuoche/',page=0):
    typeName = url.strip('/').split('/')[-1]
    finalUrl = url + 'o' + str(page)

    if(ifexistgangjiGetlistChecksheet(finalUrl)):#判斷是否爬過(guò)
        print('已經(jīng)爬過(guò)該列表')
        return

    webdata = connectTry(finalUrl)
    #webdata = requests.get(finalUrl)
    soup = BeautifulSoup(webdata.text,'lxml')


    for item in soup.select('dl.list-bigpic.clearfix dt a'):#將列表中的數(shù)據(jù)內(nèi)容按數(shù)據(jù)塊方式拆解，以方便解析
        if('x.htm' in item.get('href')):#只解析趕集標(biāo)準(zhǔn)的商品url，不標(biāo)準(zhǔn)的如轉(zhuǎn)轉(zhuǎn)等不進(jìn)行解析
            data = {
                'url':item.get('href'),
                'type':typeName,
                'catchTime':"-",

            }
            print(data)
            gangji_getInfo(data['url'])   #函數(shù)本身會(huì)判斷是否已經(jīng)爬過(guò)，這邊就沒(méi)必要進(jìn)行判斷了
            #if(not ifexistganjiUrlSheet(item.get('href'))):#如果該項(xiàng)已經(jīng)存在了，則直接跳過(guò)該項(xiàng)目
            #    gangji_getInfo(data['url'])
            #    insertganjiUrlSheet(data)
            #else:
            #    print('已經(jīng)存在')
        print('---------------------------------------------')

    insertgangjiGetlistChecksheet({'url':finalUrl})#將爬過(guò)的url插入表中，以方便下次重新開(kāi)始爬起時(shí)跳過(guò)已爬過(guò)內(nèi)容




'''                              gangji_getInfo  用來(lái)抓取商品的相關(guān)信息                            '''
def gangji_getInfo(url = 'http://bj.ganji.com/motuoche/1537729832x.htm'):
    if(ifexistgangjiGetInfoChecksheet(url)):#如果該項(xiàng)已經(jīng)存在了，則直接跳過(guò)該項(xiàng)目
        print('已經(jīng)爬過(guò)該列表')
        return

    webdata = connectTry(url)
    #webdata = requests.get(url)
    soup = BeautifulSoup(webdata.text,'lxml')

    data = {#要保存下來(lái)的數(shù)據(jù)內(nèi)容和相關(guān)結(jié)構(gòu)
        'title':soup.select('h1.title-name')[0].get_text() if soup.select('h1.title-name') else "",
        'type':[a.get_text() for a in soup.select('.crumbs.routes.clearfix a')] if soup.select('.crumbs.routes.clearfix a') else "",
        'price':None,
        'place':None,
        'phone':None,
        'date':'無(wú)日期',
        'saleday':None,
        'chense':'無(wú)數(shù)據(jù)',
        'catchDate':time.strftime("%m-%d", time.localtime()),
    }
    if soup.select('i.pr-5'):
        if(len(soup.select('i.pr-5')[0].get_text()) > 4):
            data['date'] = str(soup.select('i.pr-5')[0].get_text()).strip().split()[0]


    for item in soup.select('div.second-sum-cont ul.second-det-infor.clearfix li'):#獲取成色
        label = item.label.get_text() if item.label else ""#獲取label，以方便區(qū)分該塊是什么
        if('新舊程度：' in label):
            #print(item.get_text())
            data['chense'] = item.get_text().strip().replace('新舊程度：\n','').replace(' ','')

    for item in soup.select('ul.det-infor li'):#獲取某個(gè)塊的細(xì)分項(xiàng)目，在根據(jù)項(xiàng)目的標(biāo)簽提取信息
        label = item.label.get_text() if item.label else ""#獲取label，以方便區(qū)分該塊是什么
        if('價(jià)' in label):
            data['price'] = item.select('i.f22')[0].get_text() if item.select('i.f22') else ''
        #對(duì)交易地點(diǎn)的數(shù)據(jù)做相關(guān)處理
        if('交易地點(diǎn)' in label):
            if(item.select('a')):
                data['place'] = [a.get_text() if a else  "" for a in item.select('a')]

    print(data)
    insertganjiInfoSheet(data)
    insertgangjiGetInfoChecksheet({'url':url})








if __name__ == '__main__':
    print('main')
    #clearAllganjisheet()
    #showAllDBcount()
    gangji_getlist('http://bj.ganji.com/iphone/')
    #gangji_getInfo('http://bj.ganji.com/diannao/1991466771x.htm')
    #gangji_getInfo()
最后編輯于 ：
?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請(qǐng)聯(lián)系作者
【社區(qū)內(nèi)容提示】社區(qū)部分內(nèi)容疑似由AI輔助生成，瀏覽時(shí)請(qǐng)結(jié)合常識(shí)與多方信息審慎甄別。
平臺(tái)聲明：文章內(nèi)容（如有圖片或視頻亦包括在內(nèi)）由作者上傳并發(fā)布，文章內(nèi)容僅代表作者本人觀點(diǎn)，簡(jiǎn)書系信息發(fā)布平臺(tái)，僅提供信息存儲(chǔ)服務(wù)。

相關(guān)閱讀更多精彩內(nèi)容

友情鏈接更多精彩內(nèi)容