任務：

獲取三頁小獵短租房租信息，并寫入到數據庫中
http://sh.xiaozhu.com/search-duanzufang-p{}-0/

成果：

Snip20170525_3.png

代碼：

from bs4 import BeautifulSoup
import requests
import pymongo

client = pymongo.MongoClient('localhost',27017)
xiaozhu = client['xiaozhu']
house_info = xiaozhu['house_info']

def SexJudge(sex):
    if sex == 'member_ico':
        sex = 'man'
    else:
        sex = 'woman'
    return sex

def GetInfo(houseUrl):
    wb_data = requests.get(houseUrl)
    soup = BeautifulSoup(wb_data.text,'lxml')

    titles = soup.select('head > title')
    addresses = soup.select('span.pr5')
    rents =  soup.select('#pricePart > div.day_l > span')
    housePics = soup.select('#curBigImage')
    landlordPics = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
    sexes = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > div')


    for title,address,rent,housePic,landlordPic,sex in zip(titles,addresses,rents,housePics,landlordPics,sexes):
        data = {
            'title':title.get_text(),
            'address':address.get_text().split()[0],
            'rent':int(rent.get_text()),
            'housePic':housePic.get('src'),
            'landlordPic':landlordPic.get('src'),
            'sex':SexJudge(''.join(sex.get('class'))),
        }
    print(data)
    return data

menuUrl = ['http://sh.xiaozhu.com/search-duanzufang-p{}-0/'.format(i) for i in range(1,4)]

def GetHouseUrl(menuUrl):
    data = []
    n = 0
    for url in menuUrl:
        wb_data = requests.get(url)
        soup = BeautifulSoup(wb_data.text,'lxml')

        houseUrls = soup.select('#page_list > ul > li > a')
        for houseUrl in houseUrls:
            data.insert(-1,houseUrl.get('href'))
        n = n+1
        print('Complete Page ',n)
    return data

n=0
for houseUrl in GetHouseUrl(menuUrl):
    house_info.insert_one(GetInfo(houseUrl)) #將詳情寫入數據庫
    n = n+1
    print('Complete House ',n)


'''
(>) 大于 - $gt
(<) 小于 - $lt
(>=) 大于等于 - $gte
(<= ) 小于等于 - $lte
'''
for item in house_info.find({'rent':{'$gt':500}}):
    print(item)

參考資料：

MongoDB 教程：http://www.runoob.com/mongodb/mongodb-operators.html