#xpath:可以在xml中查找信息，對xml文檔中元素進行遍歷和屬性的提取

# xml:被設計的目的是為了傳輸數(shù)據(jù),結(jié)構(gòu)和html非常相識,是一種標記語言

"""
xpath常見的語法：
nodename 選取此節(jié)點的所有子節(jié)點
/        從根節(jié)點開始查找
//       匹配節(jié)點，不考慮節(jié)點的位置
.        選取當前節(jié)點
..       選取當前節(jié)點的父節(jié)點
a/@href        取標簽的數(shù)據(jù)
a/text()       取標簽的文本
a[@class="123"] 根據(jù)class屬性尋找標簽
a[@id="123"]    根據(jù)id屬性尋找標簽

a[@id="123"][last()]  取最后一個id為123的a標簽
a[@id="123"][postion() < 2]  取id為123的前兩個a標簽
"""
import requests
from lxml.html import etree
import re

#樣例()
#http://www.budejie.com/audio/1
#http://www.budejie.com/audio/2
#http://www.budejie.com/audio/3

def load_page_data(url):
    """
    下載器（根據(jù)分頁url獲取分頁的頁面源碼）
    :param url: 分頁url地址
    :return:
    """
    req_header = {
        'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
    }
    response = requests.get(url,headers=req_header)
    if response.status_code == 200:
        print('請求成功')
        # with open('page.html','w') as file:
        #     file.write(response.text)
        status = parse_page_data(response.text)

        if status:
            #請求下一頁數(shù)據(jù)
            pattern = re.compile('\d+')
            cur_page = re.search(pattern,response.url).group()
            next_page = int(cur_page)+1
            next_page_url = re.sub(pattern,str(next_page),response.url)

            load_page_data(next_page_url)



def parse_page_data(html):
    """
    使用xpath從html頁面源碼中提取數(shù)據(jù)
    :param html:
    :return:
    """
    #pip3 install lxml
    #使用etree.HTML()方法將html轉(zhuǎn)為xml(element對象)
    html_element = etree.HTML(html)

    autio_list = html_element.xpath('//div[@class="j-r-c"]/div[@class="j-r-list"]/ul/li')
    print(autio_list)
    print(len(autio_list))

    for autio in autio_list:
        autio_data = {}
        #取出標題
        autio_data['name'] = autio.xpath('.//a[@class="u-user-name"]/text()')[0]
        #取出內(nèi)容
        autio_data['content'] = autio.xpath('.//div[@class="j-r-list-c-desc"]/text()')[0]
        #發(fā)布時間
        autio_data['publishTime'] = autio.xpath('.//span[@class="u-time  f-ib f-fr"]/text()')[0]
        #點贊數(shù)
        autio_data['zanNum'] = autio.xpath('.//li[@class="j-r-list-tool-l-up"]/span/text()')[0]
        #差評數(shù)
        autio_data['lowNum'] = autio.xpath('.//li[@class="j-r-list-tool-l-down "]/span/text()')[0]
        #封面
        autio_data['coverImage'] = autio.xpath('.//div[@class=" j-audio"]/@data-poster')[0]
        #音頻
        autio_data['url'] = autio.xpath('.//div[@class=" j-audio"]/@data-mp3')[0]
        print(autio_data)
        download_audio_by_url(autio_data['url'],autio_data)

    if len(autio_list) > 0:
        return True
    else:
        return False

def download_audio_by_url(url,audioData):
    """
    根據(jù)銀屏url地址下載銀屏數(shù)據(jù)
    :param url:
    :param audioData:
    :return:
    """
    req_header = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
    }
    response = requests.get(url,headers=req_header)
    if response.status_code == 200:
        print(response.url,'下載成功')
        filename = response.url[-17:]
        with open('budejie/'+filename,'wb') as file:
            file.write(response.content)
            audioData['localpath'] = 'budejie/'+filename
        #將數(shù)據(jù)存儲到數(shù)據(jù)庫
        save_data_to_db(audioData)

def save_data_to_db(audio):
    print(audio)


if __name__ == '__main__':

    start_url = 'http://www.budejie.com/audio/1'
    load_page_data(start_url)