from lxml import etree
import requests
file=open('xpath.html','r',encoding='utf-8')
html=file.read()
file.close()
# print(html)
# url='陽光電影網(wǎng)首頁的網(wǎng)址'
# req=requests.get(url)
# req.encoding='gb2312'
# html=req.text
selector=etree.HTML(html)
# print(selector,type(selector))
title=selector.xpath('//title/text()')[0]
# print(title)
h1=selector.xpath('//h1/text()')[0].strip()
# print(h1)
infos=selector.xpath('//a')
for info in infos:
    a_text=info.xpath('text()')[0]
    a_href=info.xpath('@href')[0]
    # print(a_text,a_href)

# 一、div標(biāo)簽文本提取
# 將學(xué)習(xí)視頻中xpath.html文件中div標(biāo)簽下文本值
divs = selector.xpath('//div')
# “第一個div” ，“第二個div” 使用xpath結(jié)構(gòu)化提取并打印輸出
for div in divs:
    div_text = div.xpath('text()')[0].strip()
    print(div_text)
# 二、ul標(biāo)簽文本提取
uls = selector.xpath('//ul')
# 將xpath.html文件中ul標(biāo)簽下“流程” ，“xpath學(xué)習(xí)”，“流程2”文本值使用xpath結(jié)構(gòu)化提取并打印輸出
for ul in uls:
    ul_text = ul.xpath('text()')[0].strip()
    print(ul_text)
# 三、過濾標(biāo)簽
# 將xpath.html文件中的第一個div下的前3個a標(biāo)簽的文本及超鏈接 使用xpath結(jié)構(gòu)化提取，打印輸出
print(selector.xpath('//div')[0].xpath('//a/@href'))
# 四、requests模塊和lxml&xpath結(jié)合提取數(shù)據(jù)結(jié)合上節(jié)課requests模塊知識，將陽光電影網(wǎng)導(dǎo)航欄的文本及超鏈接結(jié)構(gòu)化提取
#請求url
url = 'http://www.ygdy8.com/'
#構(gòu)造headers字典
headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh;q=0.8',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    'Cookie': 'UM_distinctid=15c5ec4f20e377-0798b30518d6b4-5393662-c0000-15c5ec4f20f28b; CNZZDATA5783118=cnzz_eid%3D1150691004-1496237600-%26ntime%3D1496237600; 37cs_user=37cs10138604998; cscpvrich4016_fidx=1; 37cs_show=69',
    'Host': 'www.ygdy8.com',
    'If-Modified-Since': 'Sun, 27 Aug 2017 15:18:27 GMT',
    'If-None-Match': "802356bb471fd31:530",
    'Referer': 'https://www.baidu.com/link?url=cnL9usny1BIZEe-NZUkUbeUE4m9CM23KIysNUsVvzlK&wd=&eqid=c50f090f0001d9880000000259a2e4b0',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
}
#定義req為一個requests請求的對象
req = requests.get(url,headers=headers)
#req這個請求對象的status_code方法獲取請求的狀態(tài)碼
status_code = req.status_code
# print(status_code)
#指定網(wǎng)頁解碼方式
req.encoding = 'gb2312'
#獲取網(wǎng)頁源碼 用html變量接收 text content方法靈活運(yùn)用
html = req.text
# print(html)
selector=etree.HTML(html)
# 查詢div的id為menu下的div的class為contain下的a標(biāo)簽不包含onclick事件的所有href
print(selector.xpath("http://div[@id='menu']//div[@class='contain']//li//a[not(@onclick)]/@href"))