實現58二手商品的標題、價格、類別、瀏覽量、區域等信息的自動爬取
新增：導出數據到excel表格內
# -*- coding: utf-8 -*-
#!/use/bin/env python
import requests,time
import xlsxwriter
from bs4 import BeautifulSoup
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',
'Cookie':'myfeet_tooltip=end; bj58_id58s="VVk4K1FxVk80dUI5MDk5Nw=="; id58=c5/ns1hqLtNLk1KnJbZHAg==; als=0; 58home=sh; myfeet_tooltip=end; final_history=26088204291258; bj58_new_session=1; bj58_init_refer=""; bj58_new_uv=5; sessionid=ca5902c9-e44c-44ea-a087-bbcceaf1074b; 58tj_uuid=49f57f42-3481-42f2-9640-cb8d85a4af80; new_session=0; new_uv=6; utm_source=; spm=; init_refer='
} #防爬
workbook = xlsxwriter.Workbook('58data_1.xlsx') #新建表格
worksheet= workbook.add_worksheet('工作表1') #在表格內新建工作簿
row = 0 #從第0行開始
col = 0 #從第0列開始
def main_link(url): #從列表頁中獲取非廣告商品的詳情頁鏈接
wb_data = requests.get(url,headers=headers)
Soup = BeautifulSoup(wb_data.text,'lxml')
page_links = Soup.select('div.infocon > table > tbody > tr > td.t > a')
for page_link in page_links: #取出每一個商品的詳情頁鏈接，并將其設為函數page_urls的參數
page_urls(page_link.get('href'))
def page_urls(url): #從商品詳情頁獲取所需要的信息
wb_data = requests.get(url,headers=headers)
Soup = BeautifulSoup(wb_data.text,'lxml')
categorys = Soup.select('#nav > div > span:nth-of-type(4) > a') #商品類別
titles = Soup.select('div.info_lubotu.clearfix > div.box_left_top '
'> h1') #商品標題
prises = Soup.select('div.info_lubotu.clearfix > div.info_massege.left > '
'div.price_li > span.price_now > i') #價格
areas = Soup.select('div.info_lubotu.clearfix > div.info_massege.left >'
' div.palce_li > span > i') #區域
pageviews = Soup.select('div.info_lubotu.clearfix > div.box_left_top > '
'p > span.look_time') #頁面瀏覽量
for category, title, prise, area, pageview in zip(categorys, #將取出的信息放入字典data
titles, prises, areas, pageviews):
data = { #剔除多余信息
'category': category.get_text(),
'title': title.get_text(),
'prise': prise.get_text(),
'area': area.get_text(),
'pageview': pageview.get_text()
}
global row #引入變量row和col
global col
worksheet.write(row,col,data['category']) #表格內寫入類目
worksheet.write(row,col+1,data['title']) #表格內寫入標題
worksheet.write(row,col+2,data['prise']) #表格內寫入價格
worksheet.write(row,col+3,data['area']) #表格內寫入區域
worksheet.write(row,col+4,data['pageview']) #表格內寫入PV
row += 1 #每次循環+1行
print('Down')
time.sleep(1) #一秒爬取一次的保護機制
#取到每一頁列表的鏈接
main_urls = ['http://bj.58.com/pbdn/0/pn{}/?PGTID=0d305a36-0000-1980-da0a-4541effe84e2&ClickID=2'.format(number) for number in range(1, 21)] #第1-20頁的列表頁所有鏈接
for single_url in main_urls: #將20頁的列表頁鏈接每一個都取出來賦值給sing_url
main_link(single_url) #單個列表頁的鏈接設為函數main_link的參數
workbook.close()
