爬取某圖片網站全站代碼

貼出代碼以供大家共享，其實爬蟲很簡單，難的是突破反爬蟲，今天看了一些反爬蟲的資料，明天或后天整理一下，寫上一篇文章。

代碼如下：

import requests

from bs4 import BeautifulSoup

import os,time,random

from selenium import webdriver

user_agent_list = [

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",

"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",

"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",

"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",

"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",

"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",

"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",

"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",

"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",

"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",

"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",

"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",

"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",

"Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) App leWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53"

]

baseurl='http://www.66bb.org'

href_list=[]

name_list=[]

UA=random.choice(user_agent_list)#獲取隨機的User_Agent

headers={'User_Agent':UA}

proxies = {"http": "http://119.5.0.58:808"}

proxies = {"http": "http://219.217.80.182:1080"}

html=requests.get(baseurl,headers=headers)

html.encoding='gb2312'

Soup=BeautifulSoup(html.text,'lxml')

href_all=Soup.find('ul',class_='topline_3b').findAll('li')

for i in range(0,len(href_all)):

lanmu_href=href_all[i].find('a')['href']

lanmu_name=href_all[i].find('a').get_text()

href_list.append(lanmu_href)

name_list.append(lanmu_name)

for lanmu_num in range(0,len(href_list)):

baseurl=baseurl+href_list[lanmu_num]

start_num=0

start_page=int(start_num/20)+1

start_image=int(start_num%20)

#看了看，一共有6頁

for i in range(start_page,15):

#獲取每頁的鏈接和名稱

if i != 1:

urllink=baseurl+'list4'+str(i)+'.html'

else:

urllink=baseurl

#獲取第X頁的第XX個鏈接地址和文件夾名稱

UA=random.choice(user_agent_list)#獲取隨機的User_Agent

headers={'User_Agent':UA}

#proxies = {"http": "http://119.5.0.58:808"}

#proxies = {"http": "http://219.217.80.182:1080"}

html=requests.get(urllink,headers=headers)

html.encoding='gb2312'

Soup=BeautifulSoup(html.text,'lxml')

all_url=Soup.find(class_="fzltp").findAll('li')

#所有的dd標簽都是圖冊，而dt是欄目頭

print(u'第',i,u'頁共包含有',len(all_url),u'個圖冊，開始下載')

print(all_url[0])

print(all_url[0].find('a'))

#開始提取圖冊的鏈接地址和帖子名稱

for num in range(start_image,len(all_url)):

#獲取地址，連接后解析html，主要目的是為了獲取圖片總數

href=all_url[num].find('a')['href']

href1='http://www.66bb.org'+href

#http://www.66xx.org/ArtDD/

UA=random.choice(user_agent_list)#獲取隨機的User_Agent

headers={'User_Agent':UA}

page1html=requests.get(href1,headers=headers)

page1html.encoding='gb2312'

page1soup=BeautifulSoup(page1html.text,'lxml')

total_pages_1=page1soup.find('div',class_='tpm01').find('font',color='blue').get_text()

total_pages_1=int(total_pages_1[-3:-1])

print(u'共有',total_pages_1,u'頁圖片需要保存')

#獲取文件夾名字

dirname=all_url[num].find('img')['alt']

#建立文件夾，打開子文件夾

os.chdir("D:\\")

path = name_list[lanmu_num]+'\\'+dirname+'('+str(total_pages_1)+'P)'

isExists = os.path.exists(os.path.join("D:\\66人體藝術圖片\\",path))

if not isExists:

print(u'建了一個名字叫做', path, u'的文件夾！')

os.makedirs(os.path.join("D:\\66人體藝術圖片\\", path))

else:

print(u'名字叫做', path, u'的文件夾已經存在了！')

continue

#跳出本次循環

os.chdir("D:\\66人體藝術圖片\\"+path)

#basename=imgsrc1[-14:-5]

#尋找所有的圖片地址，打開圖片網頁，保存圖片

#最后一個頁面的網頁打不開，不是個例，應該是個普遍現象

for i in range(1,total_pages_1+1):

baseurl_2=href1+str(i) +'.html'

html_2=requests.get(baseurl_2,headers=headers)

html_2.encoding='gb2312'

soup_2=BeautifulSoup(html_2.text,'lxml')

#提取主圖的地址和名字

image_href=soup_2.find('div',class_='imgbox').find('img')['src']

image_

#這里如果普通的請求總是中斷，可以使用selenium來操縱chrome和PhantomJS

#browser=webdriver.Chrome()

#browser = webdriver.PhantomJS(executable_path="phantomjs.exe")

#browser.get(image_href)

time.sleep(1)

#imghtml=browser.page_source #這個函數獲取頁面的html

UA=random.choice(user_agent_list)#獲取隨機的User_Agent

headers={'User_Agent':UA}

try:

imgrealhtml=requests.get(image_href,headers=headers)

except:

time.sleep(5)

print(u'請稍等，5秒后重連')

UA=random.choice(user_agent_list)#獲取隨機的User_Agent

headers={'User_Agent':UA}

imgrealhtml=requests.get(image_href,headers=headers)

f = open(image_name,'ab')

f.write(imgrealhtml.content)

f.close()

print(u'第',i,u'張圖片保存完畢')

time.sleep(1)

#browser.quit()

print(u'共有',total_pages_1,u'張圖片保存完畢')

最后編輯于 ：
?著作權歸作者所有,轉載或內容合作請聯系作者
平臺聲明：文章內容（如有圖片或視頻亦包括在內）由作者上傳并發布，文章內容僅代表作者本人觀點，簡書系信息發布平臺，僅提供信息存儲服務。

推薦閱讀更多精彩內容