目標

• 爬取一頁商品數據
• 不要推廣商品和轉轉商品
• 在詳情頁獲取信息（類目 | 標題 | 發帖時間 | 價格 | 成色 | 區域 | 瀏覽量）

環境

• Python 3.5.1
• Windows7 或者 Ubuntu 14.04

結果示例

{
    'areas': ['豐臺', '-', '西客站'],
    'datetime': '2016-03-06',
    'price': '500',
    'quality': '95成新',
    'title': '北京實體店出售蘋果IPAD迷你AIR三星平板電腦',
    'type': '北京二手平板電腦',
    'views': '6103'
}

代碼

# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
import requests, time
from pprint import pprint

base_url = 'http://bj.58.com/pbdn/0/'

def get_all_links(url):
    """獲取一頁商品的鏈接，除了推廣和轉轉"""
    
    urls = []
    data = requests.get(url)
    if data.status_code == 200:
        soup = BeautifulSoup(data.text, 'lxml')
        for tb in soup.select('div#infolist > table.tbimg'):
            for item in tb.find_all("tr", class_=""):
                urls.append(item.find("a", class_="t").get('href'))
        
        return urls

def get_views(url):
    """獲取單個商品的瀏覽量"""
    
    info_id = url.split('?')[0].split('/')[-1].strip('x.shtml')
    interface = 'http://jst1.58.com/counter?infoid={}'.format(info_id)
    headers = {
        'Host': 'jst1.58.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
                        Chrome/50.0.2661.102 Safari/537.36',
        'Accept': '*/*',
        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
        'Accept-Encoding': 'gzip, deflate',
        'Referer': url,
        'Cookie': r'id58=c5/ns1djWc+KP2U1DXI4Ag==; city=bj; 58home=bj; ipcity=sh%7C%u4E0A%u6D77%7C0; \
            58tj_uuid=93b726b3-9742-4270-9d11-dcd665bea67b; new_uv=3; myfeet_tooltip=end; \
            als=0; bj58_new_uv=3; bj58_id58s="SUg0UjFZMFI0K3RSMDIyMw=="; \
            sessionid=37cd7e97-40fe-40ac-b903-d6e85ffd8690; __utma=253535702.1612735289.1466129063.1466129063.1466129063.1; \
            __utmc=253535702; __utmz=253535702.1466129063.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; \
            final_history=26364215035854%2C25245379987755%2C26062681492781; bangbigtip2=1; new_session=0; utm_source=; spm=; \
            init_refer=; bj58_new_session=0; bj58_init_refer="{}"'.format(url),
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0' 
    }
    
    resp = requests.get(interface, headers=headers)
    if resp.status_code == 200:
        return resp.text.split('=')[-1]
    else:
        return 0

def get_info(url):
    """獲取一頁中所有的商品信息"""
    
    url_list = get_all_links(url)
    
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
            Chrome/50.0.2661.102 Safari/537.36'
    }
    for link in url_list:
        data = requests.get(link, headers=headers)
        if data.status_code == 200:
            soup = BeautifulSoup(data.text, 'lxml')
            areas = soup.select('.c_25d')
            
            info = {
                'type': soup.select('div.breadCrumb span:nth-of-type(3) a')[0].get_text(),
                'title': soup.select('.mainTitle > h1:nth-of-type(1)')[0].get_text(),
                'datetime': soup.select('.time')[0].get_text(),
                'price': soup.select('.price')[0].get_text(),
                'quality': soup.select('.suUl > li:nth-of-type(2) > div:nth-of-type(2) > span:nth-of-type(1)')[0].get_text(strip=True),
                'areas': list(areas[0].stripped_strings) if len(areas) > 0 else [],
                'views': get_views(data.url)     # 使用重定向后的鏈接
            }

            pprint(info)
        time.sleep(1)

if __name__ == '__main__':
    get_info(base_url)

總結

• 由于鏈接中存在重定向的鏈接，所以需要注意要使用重定向后的鏈接
• 獲取瀏覽量需要通過其他API接口來進行獲取
• 注意網站的反爬蟲機制，可以加上請求頭或者通過代理IP