爬取 豆瓣電影我不是藥師 評(píng)論分析
#--*--coding:utf-8--*--
import requests
import json
import os
import sys
import random
import time
proxies = {
? "http": "proxy.xx.com:911",
? "https": "proxy.xx.com:911",
}
#url=r"https://movie.douban.com/subject/26752088/comments?status=P"
"""
https://movie.douban.com/subject/26752088/comments?start=0&limit=20&sort=new_score&status=P&percent_type=
https://movie.douban.com/subject/26752088/comments?start=40&limit=20&sort=new_score&status=P&percent_type=
https://movie.douban.com/subject/26752088/comments?start=60&limit=20&sort=new_score&status=P&percent_type=
"""
u1=r"https://movie.douban.com/subject/26752088/comments?start="
u2=r"&limit=20&sort=new_score&status=P&percent_type="
for i in range(20,1320,20):
? ? url=u1+str(i)+u2
? ? print(url)
? ? r=requests.get(url,proxies=proxies)
? ? print(r.status_code)
? ? html=str(r.content, encoding = "utf_8_sig")
? ? print(html)
? ? f = open('c:/users/ffan2/desktop/wbsys_pl.txt','a',encoding='utf_8_sig')
? ? f.write(html)
# coding: utf-8# In[1]:# -*- coding: utf-8 -*-import requestsimport jsonimport osimport sysimport randomimport timeimport reimport numpy as npimport pandas as pdfrom pandas import DataFrameimport matplotlib.pyplot as plt#df=pd.DataFrame(pd.read_csv('c:\\users\\ffan2\\desktop\\jd_data.csv',encoding='UTF_8'))# In[3]:html=open('c:\\users\\ffan2\\desktop\\wbsys.txt','r',encoding = "UTF_8").read()# In[158]:votes_0=re.findall(r'.*?',html)votes_1=re.sub(r'.*?','',str(votes_0))
votes=re.sub(r'',r'',str(votes_1))votes? # get all vote numberslen(votes)# In[185]:comments_0=re.findall(r'
.*',html)
comments=re.sub(r'
.*?','',str(comments_0))#comments# In[165]:comments=re.sub(r'','',str(comments))len(comments)# In[166]:comment_time_0=re.findall(r'\'',r'',str(comment_time_2))#comment_time# In[167]:user_name_0=re.findall(r'','',str(user_name_0))
user_name=re.sub('<','',str(user_name_1))
#user_name
# In[154]:
a={'user_name':user_name,'comment_time':comment_time,'votes':votes,'comments':comments}
print(len(a['comments']))
print(len(a['votes']))
print(len(a['comment_time']))
print(len(a['user_name']))
# In[173]:
table=pd.DataFrame(a[comments])
#table.to_csv('c:\\users\\ffan2\\desktop\\wsys.csv',encoding = "utf_8_sig")
#df=pd.DataFrame(pd.read_csv('c:\\users\\ffan2\\desktop\\wsys.csv',encoding='UTF_8_sig'))
#print(table.shape)
#使用jieba 分詞 分析rank~
word_str=''.join(comments)
word_rank=jieba.analyse.extract_tags(word_str, topK=20, withWeight=True, allowPOS=())
print(word_rank)
word_rank = pd.DataFrame(word_rank,columns=['word','rank'])
word_rank.set_index('rank')
rank?word
0.159256電影
0.063695徐崢
0.056682題材
0.052965煽情
0.046412現(xiàn)實(shí)
0.042310王傳君
0.040851藥神
0.033984一部
0.032249達(dá)拉斯
0.028569這部
0.026092片子
0.026046希望
0.025504導(dǎo)演
0.025464真實(shí)
0.025396故事
0.024536劇情
0.024384中國(guó)
0.024108社會(huì)
0.023750國(guó)產(chǎn)片
0.023353演員
