實戰計劃0430-石頭的練習作業
練習的要求
課程要求抓取的內容：
實現效果如下
運行結果
相關代碼
__author__ = 'daijielei'
'''
11課時練習，從小豬短租處抓取一堆租房詳情
'''
from bs4 import BeautifulSoup #解析html文檔用
import requests #抓取html頁面用
import time
#urls用來制定抓取范圍，header該處無用處
header = {
'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
'Host':'bj.xiaozhu.com'
}
#getlist用來抓取對應網頁上的每個列表的跳轉url
#測試用url = "http://bj.xiaozhu.com/search-duanzufang-p1-0/"
def getlist(url):
webData = requests.get(url,headers=header)
soup = BeautifulSoup(webData.text,'lxml')
houselists = soup.select('ul > li > a')
for houselist in houselists:
listUrl = str(houselist.get('href'))
if(listUrl.find('http://bj.xiaozhu.com/fangzi/')!=-1):#確認鏈接內容為詳情頁，則進行詳情頁信息抓取
#print(listUrl)
getInfoPage(listUrl)
#getInfoPage用來抓取詳情頁里的相關信息，如標題、地址、金額、房屋圖片、擁有人圖片等等
#測試用url = "http://bj.xiaozhu.com/fangzi/525041101.html"
def getInfoPage(url):
webData = requests.get(url,headers=header)
soup = BeautifulSoup(webData.text,'lxml')
time.sleep(2)
titles = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em')
address = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5')
pays = soup.select('#pricePart > div.day_l > span')
houseimages = soup.select('#curBigImage')
ownerimages = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
ownernames = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
ownerSexs = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span')
for title,addres,pay,houseimage,ownerimage,ownername,ownerSex in zip(titles,address,pays,houseimages,ownerimages,ownernames,ownerSexs):
data = {
'title':title.get_text(),
'addres':addres.get_text(),
'pay':pay.get_text(),
'houseimage':houseimage.get('src'),
'ownerimage':ownerimage.get('src'),
'ownername':ownername.get_text(),
'ownerSex':'female'
}
if('member_boy_ico' in str(ownerSex['class'])):#該處用來判斷是男性還是女性，男性是通過標簽來區分
data['ownerSex'] = "male"
print(data)
def startCatch():
urls = ["http://bj.xiaozhu.com/search-duanzufang-p{}-0/".format(str(i)) for i in range(1,3,1)]
for url in urls:
getlist(url)
#入口，該語句只有直接執行時會被調用
if __name__ == '__main__':
startCatch()
筆記、想法、總結
1、這個代碼是之前寫的了，所以顯得比較亂，尤其是抓取的部分現在寫的話我就不會用這種寫法了
2、這里面主要有3個函數，功能還是比較清晰，先爬取列表，再進列表里去爬取數據：
def startCatch(): #用來便利列表，會調用getlist
def getlist(url): #用來抓取列表的數據，會調用getInfoPage
def getInfoPage(url):#抓取詳情頁的數據
3、比較難抓的數據只有性別信息，因為不是直接能抓取的值，而是要做一下判斷：
在html中為如下,是用span里的class來區別男女，男為member_boy_ico、女為member_girl_ico：
<h6>
<a class="lorder_name" title="ocheese" target="_blank">ocheese</a>
<span class="member_girl_ico"></span>
</h6>
代碼中就是做一下判斷即可
if('member_boy_ico' in str(ownerSex['class'])):#該處用來判斷是男性還是女性，男性是通過標簽來區分
data['ownerSex'] = "male"
