一、數(shù)據(jù)存儲

#1)存入json
import json
def write_to_json(data):
    # 把數(shù)據(jù)整合json支持的類型
    json_list = []
    for houses in data:
        for house in houses:
            json_list.append(house)

    with open('lianjian.json','w') as fp:
        fp.write(json.dumps(json_list，index=2,ensure_ascii=False))
        # .dumps()方法:將json對象轉(zhuǎn)化為字符串
        # 參數(shù)indent:表示縮進(jìn)個數(shù)
        # 參數(shù)ensure_ascii:是否將中文字符轉(zhuǎn)化為Unicode字符

import csv
#2) 存入csv
def write_to_csv(data):
    # 在寫csv的時候，需要把data整合成一個二維列表
    # 定義一個大的列表，用于保存所有的房屋信息
    csv_items = []
    for houses in data:
        for house in houses:
            # house是字典，按照鍵值的形式存儲了每個房屋的信息，取出值寫入列表
            item = []
            for k,v in house.items():
                item.append(v)
            csv_items.append(item)

    # 寫入csv
    with open('lianjjia.csv','w') as fp:
        # 用fp來創(chuàng)建一個csv的寫對象
        w = csv.writer(fp)
        w.writerow(['title','house','position','totalPrice','unitPrice','img'])
        w.writerows(csv_items)
        # .writerows()方法：同時寫入多行，參數(shù)是個二維列表· 4
        
import pymysql
#3) 存入數(shù)據(jù)庫
def write_to_mysql(data):
    # 創(chuàng)建一個mysql數(shù)據(jù)庫的鏈接
    db = pymysql.connect(host='127.0.0.1',port=3306,user='root',password='123456',db = 'lianjia',charset='utf8')
    # 創(chuàng)建一個游標(biāo)，用于解析sql語句
    cursor = db.cursor()

    # 創(chuàng)建sql語句
    for houses in data:
        for house in houses:
            sql = 'INSERT INTO ershoufang VALUES (NULL,"%s","%s","%s","%s","%s","%s",)'%(house['title'],house['house'],house['position'],house['totalPrice'],house['nuitPrince'],house['img'])

            # 解析并提交sql語句
            cursor.execute(sql)
            db.commit()

    # 關(guān)閉游標(biāo)和數(shù)據(jù)庫連接
    cursor.close()
    db.close()

二、ip代理

from  urllib import request
import requests
url = 'https://www.baidu.com/s?ie=utf-8&f=3&rsv_bp=1&tn=baidu&wd=ip&oq=ip&rsv_pq=b254df33000238fe&rsv_t=caee8Radj5kHT5OB1roVV9axqOakQtWZzVH9BYWRWLXkJtyQBHfRRlcDylg&rqlang=cn&rsv_enter=0&rsv_sug3=1&rsv_sug1=1&rsv_sug7=100&prefixsug=ip&rsp=0&rsv_sug4=1224'

headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
    }

##########################################################3
# 創(chuàng)建一個請求對象
# request_obj = request.Request(url=url,headers=headers)

# 配置代理
# handler = request.ProxyHandler({"https":"113.200.56.13:8010"})

# 創(chuàng)建一個opener攜帶代理handler(h)

# opener = request.build_opener(handler)
# res = opener.open(request_obj)
####################################################################

res = requests.get(url=url,headers=headers,proxies = {"https":"113.200.56.13:8010"})

with open('./ip.html','wb') as fp:
    fp.write(res.content)
    fp.close()

三、模擬登陸

• 模擬古詩文網(wǎng)登陸
  </br>
  使用Session狀態(tài)保持

import requests
from lxml import etree


headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
    }

# 登錄頁的url
page_url = 'https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx'

# 登錄需要提交的內(nèi)容:用戶名、密碼、靜態(tài)表單、兩個token、驗(yàn)證碼

# 經(jīng)過分析，兩個token和驗(yàn)證碼是在訪問登錄頁的時候動態(tài)生成
# log_html = requests.get(page_url)
s =requests.Session()    # 狀態(tài)保持
log_html = s.get(page_url,headers=headers)
# 訪問登錄頁的時候也要用到session，因?yàn)閠oken值保存在后臺,Session可以記錄后臺token值


# 獲取兩個token
login_tree = etree.HTML(log_html.text)

token_a = login_tree.xpath('//*[@id="__VIEWSTATE"]/@value')
token_b = login_tree.xpath('//*[@id="__VIEWSTATEGENERATOR"]/@value')

# 獲取驗(yàn)證碼圖片
code_url = 'https://so.gushiwen.org'+login_tree.xpath('//*[@id="imgCode"]/@src')[0]
print(code_url)
# 下載驗(yàn)證碼
code_info = s.get(code_url)
with open('./code.png','wb') as fp:
    fp.write(code_info.content)
    fp.close()

# 驗(yàn)證碼可以有以下幾種處理機(jī)制，用第三方的ai平臺接口來識別、自己訓(xùn)練AI、人工識別
code = input('請輸入你看到的驗(yàn)證碼')

# 登錄信息提交的信息url
log_url = 'https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx'

# 請求體
data = {
'__VIEWSTATE': token_a,
'__VIEWSTATEGENERATOR': token_b,
'from': 'http://so.gushiwen.org/user/collect.aspx',
'email': 'fanjianbo666@163.com',
'pwd': '12345678',
'code': code,
'denglu': '登錄'
}

result = s.post(url=log_url,data=data)
print(result.text)

• 模擬登陸chinaunix網(wǎng)

import requests
from bs4 import BeautifulSoup

headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
    }


# 登錄頁的url

login_page = 'http://bbs.chinaunix.net/member.php?mod=logging&action=login&logsubmit=yes'
# 訪問登錄頁面
s = requests.Session()
page_html = s.get(login_page,headers=headers)
soup = BeautifulSoup(page_html.text,'lxml')
action = soup.select('form.cl')[0].get('action')
formhash = soup.select('[name="formhash"]')[0].get('value')

# formhash、action是動態(tài)生成


# 登錄接口需要動態(tài)獲取
login_url = 'http://bbs.chinaunix.net/'+action

data = {
    'formhash':formhash,
    'referer':'http://bbs.chinanuix.net/./',
    'username':'Mrfan666',
    'password':'f12345678',
    'loginsubmit':'true',
    'returen_type':''

}

r = s.post(url=login_url,headers=headers,data=data)
print(r.text)