之前在麥子學(xué)院看了黃老師關(guān)于爬蟲(chóng)的基礎(chǔ)課程，自己試著練習(xí)一下，不過(guò)和黃老師的有些區(qū)別，我使用了BeautifulSoup包，等于是改寫(xiě)了一下黃老師的代碼吧。

以下是實(shí)現(xiàn)代碼，直接上干貨

# coding:utf-8

import urllib2,re
from bs4 import BeautifulSoup
import bs4

def retrive_tangshi_300():
    url = 'http://www.gushiwen.org/gushi/tangshi.aspx'
    r = urllib2.urlopen(url)
    soup = BeautifulSoup(r.read(),'html.parser',from_encoding='utf-8')

    # 通過(guò)select選取標(biāo)簽內(nèi)容、地址
    #tags = soup.select('div a')
    #for tag in tags:
    #    print tag['href']

    shige_list = []
    current_poem = {}

    tags = soup.find_all('div', class_ = "guwencont2")
    for tag in tags:
        #print tag.a['href']
        for t in tag.children:
            #print t,type(t)
            if type(t) == bs4.element.Tag:
                pattern = re.compile(r'(.*)\((.*)\)')
                m = pattern.match(t.string)
                if m:
                    current_poem['url'] = t['href']
                    current_poem['title'] = m.group(1)
                    current_poem['author'] = m.group(2)
                    shige_list.append(current_poem)
                    current_poem = {}
    return shige_list



if __name__ == '__main__':
    r = retrive_tangshi_300()
    print 'url: ',r[0]['url'],'\n 作者：',r[0]['author'],'\n 題目：',r[0]['title']