包包配置

import requests
import ast
import json

import sys
reload(sys)
sys.setdefaultencoding('utf-8')

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/44.0',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
    'Accept-Encoding': 'gzip, deflate',
    'Connection': 'keep-alive'}

當網頁可以找到相應的json數據的時候：

url = ""
# 在網頁header頁面拉到最下面然后找到postdata部分
postdata= {
}
import requests
# 請求
def get_html(url):
    try:
        r = requests.get(url, headers={'User-Agent':'Mozilla/5.0'}, timeout=30)
        r.raise_for_status() # throw HTTPError if the status code is not 200
        r.encoding = r.apparent_encoding # handling encoding issue
        return r.text
    except:
        return "Error: something is Wrong!"
result = requests.post(url,data=postdata, headers=headers).text

# 對返回的string進行按照 自己的 需求進行清洗
result = result.replace('\n','')[4:]
result = result.replace('null,', '')

# 將list  dic 類型的string轉成 list dic
clean_result = ast.literal_eval(result)

print json.dumps(clean_result).decode("unicode-escape")

當需要用到解析網頁的時候：

一般來說會用到Beautiful Soup: https://www.crummy.com/software/BeautifulSoup/bs4/doc.zh/
BeautifulSoup庫是用來解析、遍歷、維護“標簽樹”的功能庫（t.name,t.string,t.attrs）
另一種方法是RE: https://docs.python.org/3/library/re.html

def getMobileInfo(base_url, page):
    result = []

    for i in range(0,page,1):
        # 拼接url
        url = ''.join([base_url, str(i), '.html'])
        html = requests.get(url, headers=headers).text
        beautyHtml = BeautifulSoup(html, 'lxml')#解析式的一種，錯誤最少，最全的
        ＃一般來說都會尋找相應的
        div_list = beautyHtml.find_all('div', attrs={
            'class': 'feeds-item'})

        if div_list:
            for item in div_list:
                # 取得tag屬性的值
                href = item.div.h3.a['href']
                # 取得tag的值
                name = item.div.h3.a.text.encode('utf-8')
                reg = '201\d年\d+月'
                regResul = re.findall(reg, name, re.S)
                if len(regResul) != 0:
                    print '___Name:', name
                    print '___ADD', href
                    result.append(href)
    return result
## get all links in the page
link_list = [link.get('href') for link in soup.find_all('a')]
for link in link_list:
    if link is not None and link[:4] == 'http': # Note: lazy evaluation
        external_links.append(link)
[link for link in link_list if l is not None and l.startswith('http')]

網頁的結構和re相關文檔

各種tag等的含義：https://www.w3schools.com/tags/
大部分的瀏覽器會將HTML解析成DOM:https://www.w3.org/DOM/
正則表達式：https://regexone.com/ ，https://docs.python.org/3/howto/regex.html
正則表達式測試：http://regexr.com/
代理池使用:/wenshu_scrach/將文件夾和smilerequest.py引入即可（先cd到ip文件夾然后run起來）

定制好的函數

list抓取

def getListData(outputfile, *args):
    url = ''.join(args)
    print url
    html = requests.get(url, headers=headers).text
    beautyHtml = BeautifulSoup(html, 'lxml')

    # 第三步：最重要的定制規則
    # 搜索tag
    tag = beautyHtml.find('div', attrs={'class': 'bottom'})
    tags = beautyHtml.find_all('div')
    tags = beautyHtml.find_all(id='html')
    sub_tags = tag.find_all('div')

    # 獲取tag的值
    tag_value = tag.div.string  # type: bs4.element.NavigableString 也是一種unicode, 可以用str() 進行轉換
    tag_value = tag.head.text  # type: bs4.element.NavigableString 也是一種unicode, 可以用str() 進行轉換

    # 獲取property的值
    property_value = tag['class'].string


if __name__ == '__main':
    # 第一步：確定好輸出文件
    inputfliePath = '/Users/i309929/Desktop/input.txt'
    outputfile = open(inputfliePath, 'w')

    # 第二步：定制url
    url = 'http://www.aqistudy.cn/historydata/index.php'
    getListData(outputfile, url)

    outputfile.close()

表格抓取

def getExcelFromWebTable(outputSheet, *keyword):
    url = ''.join(keyword)
    print url
    html = requests.get(url, headers=headers).text
    beautyHtml = BeautifulSoup(html, 'lxml')

    tables = beautyHtml.find_all('table')
    if tables:
        print "一共找到 " + str(len(tables)) + " 表格"
        print "默認選取第一個表格......."
        first_table = tables[0]
        trs = first_table.find_all('tr')

        if trs:
            row_count = len(trs)
            # print "行數：" + str(row_count)
            for i in xrange(0, row_count):
                global  start_row
                tr = trs[i]
                tds = tr.find_all('td')
                if tds==None:
                    tds = tr.find_all('th')
                col_count = len(tds)
                # print "列數" + str(col_count)
                for j in xrange(0, col_count):
                    td = tds[j]
                    text = td.text
                    outputSheet.write(start_row, j + 1, label=text)
                start_row = start_row + 1
        else:
            print "表格tr中沒有數據"
    else :
        print url + " 中沒有表格格式數據"

if __name__ == '__main__':
    '''
    ------------------------------------------------------------------------------------------------
    從text文件中中每一行讀取關鍵字變量進行遍歷
    '''
    # inputFilePath = '/Users/i309929/Desktop/cities.txt'
    # queryKeywords = open(inputFilePath, 'r')
    #
    # outputFile = ExcelWrite.Workbook(encoding='utf-8')
    # outputSheet = outputFile.add_sheet("output_sheet", cell_overwrite_ok=True)
    #
    # for keyword in queryKeywords:
    #     global start_row
    #     start_row = start_row + 1
    #     outputSheet.write(start_row, 0, label=keyword)
    #     keyword = string.rstrip(keyword)
    #     print '-------------keyword: ' + str(keyword) + ' ----: '
    #     baseURL = 'http://www.aqistudy.cn/historydata/monthdata.php?city='
    #     getExcelFromWebTable(outputSheet, baseURL, str(keyword))
    #     sleep(1)
    #
    # queryKeywords.close()
    # outputFile.save('/Users/i309929/Desktop/output.xls')

    '''
    ------------------------------------------------------------------------------------------------
    直接進行遍歷
    '''
    outputFile = ExcelWrite.Workbook(encoding='utf-8')
    outputSheet = outputFile.add_sheet("output_sheet", cell_overwrite_ok=True)

    getExcelFromWebTable(outputSheet, 'http://tianqihoubao.com/weather/top/beijing.html')

    outputFile.save('/Users/i309929/Desktop/output.xls')