結果
霉霉.png
我的代碼
__author__ = 'CP6'
# -*- coding:utf-8 -*-
from bs4 import BeautifulSoup
import requests
import time
# proxies = {"http": "207.62.234.53:8118"}
urls = ['http://weheartit.com/inspirations/taylorswift?scrolling=true&page={}'.format(str(i)) for i in range(1, 5)]
proxies = {'HTTP': '123.56.28.196:8888'}
headers = {
'UserAgent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.21 Safari/537.36'
}
def download(url):
r = requests.get(url, headers=headers, proxies=proxies)
if r.status_code != 200:
return
# http://data.whicdn.com/images/254100583/superthumb.jpg
filename = url.split('/')[4]
target = './{}.jpg'.format(filename)
# print(target)
with open(target, "wb") as fs:
fs.write(r.content)
print("%s => %s" % (url, target))
def main():
for single_url in urls:
wb_data = requests.get(single_url, headers=headers, proxies=proxies)
if wb_data.status_code != 200:
continue
soup = BeautifulSoup(wb_data.text, 'lxml')
time.sleep(3)
imgs = soup.select('a.js-entry-detail-link > img')
for img in imgs:
src = img.get('src')
download(src)
if __name__ == '__main__':
main()
總結
-
難點1：代理
使用瀏覽器的UserAgent防止反爬取，使用代理來訪問一些墻外的網站。
proxies = {'HTTP': '123.56.28.196:8888'}
headers = {
'UserAgent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.21 Safari/537.36'
}
r = requests.get(url, headers=headers, proxies=proxies)
代理的鏈接：proxy_pool
-
難點2 :獲取img的Selector
Paste_Image.png
imgs = soup.select('a.js-entry-detail-link > img')
-
難點3: 獲取img文件名
str.split('/') str被分割之后，會生成一個list，這里方便起見，簡單的用了list[4]獲取了文件名
// http://data.whicdn.com/images/254100583/superthumb.jpg
filename = url.split('/')[4]
['http:', '', 'data.whicdn.com', 'images', '247108528', 'superthumb.jpg']
-
難點4: 寫文件-保存圖片到本地
其實就是把url解析后的二進制寫入target文件中（文件類型已寫為.jpg），也就是把圖片保存到了本地， r.content獲取響應后的二進制內容
r = requests.get(url, headers=headers, proxies=proxies)
target = './{}.jpg'.format(filename)
with open(target, "wb") as fs:
fs.write(r.content)
py語法
-
strip()函數，random.choice()函數
strip(rm) 刪除s字符串中開頭、結尾處，位于 rm（默認為空格）刪除序列的字符
strip.png
** random.choice(seq) 從序列中獲取一個隨機元素**
import random
random.choice(range(10)) #輸出0到10內隨機整數
random.choice(range(10,100,2)) #輸出隨機值[10,12,14,16...]
random.choice("I love python") #輸出隨機字符I,o,v,p,y...
random.choice(("I love python")) #同上
random.choice(["I love python"]) #輸出“I love python”
random.choice("I","love","python") #Error
random.choice(("I","love","python")) #輸出隨機字符串“I”，“love”，“python”
random.choice(["I","love","python"]) #輸出隨機字符串“I”，“love”，“python”
引用文章地址：
strip函數
Python random模塊
-
request.get(url, headers=headers, proxies=proxies, timeout=timeout)
payload = {'key1': 'value1', 'key2': 'value2'}
r = requests.get("http://httpbin.org/get", params=payload)
