mongodb的使用
- step1 處理url，先得到各個page的url, 用geturls()得到每個page中各個房源的url, 再用soup解析得到每個房源的相關信息，依次存入數據庫
- step2 從數據庫查找>500的房源數據
- 順便實現了斷點續傳
#!usr/bin/env python
#_*_ coding: utf-8 _*_
#
# 在mongodb中篩選房源
#
from bs4 import BeautifulSoup
import requests
import os
import urllib
import pymongo
from mongoconn import mongoset
def geturls(url):
url_data = requests.get(url)
soup = BeautifulSoup(url_data.text, "lxml")
pagelist = soup.select('#page_list > ul > li > a')
urllist = [item.get('href') for item in pagelist]
return urllist
def getemtext(element):
return element.get_text().strip()
def getgender(element):
if element.get("class") == ["member_boy_ico"]:
return 'male'
elif element.get("class") == ["member_girl_ico"]:
return 'female'
else:
return ''
pass
def get_target_info(url):
url_data = requests.get(url)
soup = BeautifulSoup(url_data.text, "lxml")
title = soup.select('div.pho_info > h4 > em')
address = soup.select('div.pho_info > p > span.pr5')
image = soup.select('div.pho_show_big > div > img')
price = soup.select('div.bg_box > div.day_top > div > span')
lordname = soup.select('div.bg_box > div > div > h6 > a.lorder_name')
gender = soup.select('div.bg_box > div > div > h6 > span')
lordpic = soup.select('div.bg_box > div > div.member_pic > a > img')
data = {
"title": getemtext(title[0]),
"address": getemtext(address[0]),
"image": image[0].get('src'),
"price": getemtext(price[0]),
"lordname": getemtext(lordname[0]),#.get_text(),
"gender": getgender(gender[0]),
"lordpic": lordpic[0].get('src')
}
return data
if __name__ == '__main__':
tinfo = mongoset('xiaozhu', 'info')
tinfo.create_index([('title', pymongo.DESCENDING), ('address', pymongo.DESCENDING)], unique=True)
### step1 save data into mongodb
urls = ["http://sh.xiaozhu.com/search-duanzufang-p{}-0/".format(pageid) for pageid in range(1, 4)]
print(urls)
for url in urls:
urllist = geturls(url)
for urlitem in urllist:
data = get_target_info(urlitem)
print({'title': data['title']})
try:
tinfo.insert_one(data)
except pymongo.errors.DuplicateKeyError:
tinfo.update_one({'title': data['title']}, {'$set': {'price': data['price']}})
except:
pass
### step2 get target info
for info in tinfo.find({'price': {'$gt': '500'}}):
#if int(info['price']) >= 500:
print(info)
部分輸出信息如下
{'gender': 'female', 'lordpic': 'http://image.xiaozhustatic1.com/21/4,0,2,9380,374,374,194746bc.jpg', '_id': ObjectId('5775150484a5fd48784276e0'), 'image': 'http://image.xiaozhustatic1.com/00,800,533/4,0,81,10040,1798,1200,a0fc0b77.jpg', 'lordname': '空白色', 'address': '上海市閘北區止園路99弄1號樓', 'title': '人民廣場 外灘 地鐵零距離旅游溫馨便利兩居室', 'price': '599'}
{'gender': 'female', 'lordpic': 'http://image.xiaozhustatic1.com/21/4,0,2,9380,374,374,194746bc.jpg', '_id': ObjectId('5775150684a5fd48784276e3'), 'image': 'http://image.xiaozhustatic1.com/00,800,533/5,0,36,3151,1798,1200,62875638.jpg', 'lordname': '空白色', 'address': '上海市閘北區寶昌路399弄3號樓', 'title': '近外灘人民廣場上海火車站地鐵站的豪華公寓', 'price': '599'}
{'gender': 'female', 'lordpic': 'http://image.xiaozhustatic1.com/21/2,0,7,3130,260,260,1edf95cd.jpg', '_id': ObjectId('5775150c84a5fd48784276ed'), 'image': 'http://image.xiaozhustatic1.com/00,800,533/2,0,74,4017,1800,1200,b920d1da.jpg', 'lordname': 'glorymedia', 'address': '上海市閔行區中春路新龍路', 'title': '近虹橋機場、高鐵，花園鋼琴房', 'price': '1088'}
{'gender': 'female', 'lordpic': 'http://image.xiaozhustatic1.com/21/6,0,64,1269,250,250,8cd6afee.jpg', '_id': ObjectId('5775150e84a5fd48784276f0'), 'image': 'http://image.xiaozhustatic1.com/00,800,533/3,0,93,4657,1800,1200,38973fc6.jpg', 'lordname': '夏小G', 'address': '上海市嘉定區嘉寶夢之灣', 'title': '古鎮多彩設計感北歐風格民居 電影拍攝片場', 'price': '550'}
{'gender': 'female', 'lordpic': 'http://image.xiaozhustatic1.com/21/1,0,2,3091,260,260,bd3907dd.jpg', '_id': ObjectId('5775151684a5fd48784276fd'), 'image': 'http://image.xiaozhustatic1.com/00,802,533/5,0,50,1993,1300,865,83b4ddfc.jpg', 'lordname': '介介', 'address': '上海市黃浦區中山南路398號', 'title': '獨享外灘豪宅江景房 東方明珠 豫園', 'price': '799'}
{'gender': '', 'lordpic': 'http://image.xiaozhustatic1.com/21/2,0,30,4168,375,375,668e4b62.jpg', '_id': ObjectId('5775151c84a5fd4878427706'), 'image': 'http://image.xiaozhustatic1.com/00,800,533/6,0,15,2889,1798,1200,4573a3c7.jpg', 'lordname': '王湘', 'address': '上海市徐匯區襄陽南路306弄', 'title': '法國手藝人打造/法租界花園洋房/電影拍攝地', 'price': '538'}
{'gender': 'male', 'lordpic': 'http://image.xiaozhustatic1.com/21/4,0,4,9896,329,329,b702853f.jpg', '_id': ObjectId('5775151e84a5fd4878427709'), 'image': 'http://image.xiaozhustatic1.com/00,800,533/4,0,85,10672,1798,1200,d35465af.jpg', 'lordname': 'siyan5', 'address': '上海市徐匯區淮海中路1200弄', 'title': '淮海中路靜安寺1/2/7/10號地鐵兩房兩衛', 'price': '698'}
數據表中保存的數據截圖
Screen Shot 2016-06-30 at 8.48.17 PM.png
總結
- 有的房源沒有房主的性別信息，將gender屬性置空
- from mongoconn import mongoset 引入的函數如下：
#!usr/bin/env python
#_*_ coding: utf-8 _*_
#
# connect mongodb
import pymongo
def mongoset(db, table):
client = pymongo.MongoClient('localhost', 27017)
data = client[db]
sheet = data[table]
return sheet
- mongo更新數據的用法：update_one
- mongo設置主鍵的方法：create_index
