Date:2016-10-15
By:Black Crow

前言：

本次作業(yè)為第二周大作業(yè)。
作業(yè)思路分為三部分：第一部分是爬取版塊，第二部分爬取單個(gè)版塊內(nèi)的產(chǎn)品鏈接，第三部分是通過(guò)產(chǎn)品鏈接爬取產(chǎn)品詳情。

作業(yè)效果：

產(chǎn)品詳情.png

我的代碼：

20161015代碼PART1：爬取頁(yè)面

from bs4 import BeautifulSoup
import requests
from pymongo import MongoClient
client =MongoClient('localhost',27017)
ganji = client['ganji']
page_urls =ganji['page_urls']
def get_page_list():
page = 'http://bj.ganji.com/wu/'
path = 'http://bj.ganji.com/'
wb_data = requests.get(page)
wb_data.encoding = 'utf-8'
soup = BeautifulSoup(wb_data.text, 'lxml')
page_lists = soup.select('dd > a:nth-of-type(1)')
# page_url_lists = []
for page_list in page_lists:
page_name = page_list.get('href').split('/')[1]
# print(page_name)
page_url = path + page_name
if page_name == 'zibubaojian':
pass
else:
page_urls.insert_one({'url':page_url})
# print(page_url)
get_page_list()

#####20161015代碼PART2：爬取產(chǎn)品鏈接
>```
from bs4 import BeautifulSoup
from multiprocessing import Pool
import requests,time
# from content import list#從另外一個(gè)文件中導(dǎo)入，須保證為同一文件夾下，后面修改為從數(shù)據(jù)庫(kù)調(diào)取
from pymongo import MongoClient
client =MongoClient('localhost',27017)
ganji = client['ganji']
page_urls = ganji['page_urls']
product_list =ganji['product_list1']
def counter(i=[0]):
    next = i[-1] + 1
    i.append(next)
    return i[-1]
def get_product_urls(channel,page):
    page_url ='{}/o{}/'.format(channel,str(page))#page記得轉(zhuǎn)為str
    page_data =requests.get(page_url)
    page_soup =BeautifulSoup(page_data.text,'lxml')
    product_urls = page_soup.select('td.t > a')
    for product_url in product_urls:
        url =product_url.get('href').split('?')[0]#shtml后面有一大串字符，可以刪掉
        if url == None:#抓不到就跳過(guò)
            pass
        else:
            product_list.insert_one({'url':url})#應(yīng)該以規(guī)范的格式傳到數(shù)據(jù)庫(kù)中，否則會(huì)報(bào)錯(cuò)
            print(counter())
            time.sleep(1)
#單線程的代碼
# def get_list():
#     for item in list:
#         for i in range(1, 100):
#             get_product_urls(item,i)
#             time.sleep(1)
def get_list(channel):
    for i in range(1,100):
        get_product_urls(channel,i)
if __name__ == '__main__':
  list =[]
  for item in page_urls.find():
      list.append(item['url'])
      # print(type(list))
  pool = Pool()
  # pool = Pool(processes=6)
  pool.map(get_list,list)#第二個(gè)參數(shù)是list

20161015代碼PART3：爬取產(chǎn)品詳情

from bs4 import BeautifulSoup
from multiprocessing import Pool
import requests,time
from pymongo import MongoClient
client =MongoClient('localhost',27017)
ganji = client['ganji']
page_urls = ganji['page_urls']
product_list =ganji['product_list']
product_info = ganji['product_info']

url = 'http://zhuanzhuan.ganji.com/detail/783666465658650628z.shtml'

def counter(i=[0]):
next = i[-1] + 1
i.append(next)
return i[-1]
def get_product_info(url):
wb_data =requests.get(url)
soup =BeautifulSoup(wb_data.text,'lxml')
titles = soup.select('h1')
now_prices = soup.select('div.price_li > span > i')
original_prices = soup.select('b.price_ori')
areas = soup.select('div.palce_li > span > i')
seller_names =soup.select('p.personal_name')
# print(seller_names)
for title,now_price,original_price,
area,seller_name in zip(titles,now_prices,
original_prices,areas,seller_names):
data ={
'title':title.get_text(),
'now_price':now_price.get_text(),
'original_price':original_price.get_text(),
'area':area.get_text(),
'seller_name':seller_name.get_text(),
'url':url
}
# print(data)
product_info.insert_one(data)
print(counter())
time.sleep(1)
if name == 'main':
list =[]
for item in product_list.find():
list.append(item['url'])
# print(type(list))
pool = Pool()

pool = Pool(processes=6)

pool.map(get_product_info,list)#第二個(gè)參數(shù)是list

####總結(jié)：
>1. pool()函數(shù)添加進(jìn)去了，比單進(jìn)程要快很多，但怕被封，還是設(shè)置了暫停時(shí)間；
2. pool()函數(shù)第二個(gè)參數(shù)是list，直接從數(shù)據(jù)庫(kù)提取出來(lái)的數(shù)據(jù)暫時(shí)不知道如何表達(dá)，所以直接是添加到了一個(gè)list里面，后面再看是否有更簡(jiǎn)單的表達(dá)；
3. 調(diào)用pool函數(shù)的時(shí)候前面的參數(shù)傳導(dǎo)一定要確保正確，本次作業(yè)在數(shù)據(jù)庫(kù)插入部分坑了好久，好在后來(lái)找出了問(wèn)題。
4. 爬了6萬(wàn)條，數(shù)據(jù)里有重復(fù)，還得查閱和解決MongoDB查重的問(wèn)題。