哎，太晚了，有空再寫注釋

Paste_Image.png

首先是隊列文件mongodb_queue的代碼,復制臥槽哥的

from datetime import datetime, timedelta
 +from pymongo import MongoClient, errors
 +
 +class MogoQueue():
 +
 +    OUTSTANDING = 1 ##初始狀態
 +    PROCESSING = 2 ##正在下載狀態
 +    COMPLETE = 3 ##下載完成狀態
 +
 +    def __init__(self, db, collection, timeout=300):##初始mongodb連接
 +        self.client = MongoClient()
 +        self.Client = self.client[db]
 +        self.db = self.Client[collection]
 +        self.timeout = timeout
 +
 +    def __bool__(self):
 +        """
 +        這個函數，我的理解是如果下面的表達為真，則整個類為真
 +        至于有什么用，后面我會注明的（如果我的理解有誤，請指點出來謝謝，我也是Python新手）
 +        $ne的意思是不匹配
 +        """
 +        record = self.db.find_one(
 +            {'status': {'$ne': self.COMPLETE}}
 +        )
 +        return True if record else False
 +
 +    def push(self, url, title): ##這個函數用來添加新的URL進隊列
 +        try:
 +            self.db.insert({'_id': url, 'status': self.OUTSTANDING, '主題': title})
 +            print(url, '插入隊列成功')
 +        except errors.DuplicateKeyError as e:  ##報錯則代表已經存在于隊列之中了
 +            print(url, '已經存在于隊列中了')
 +            pass
 +    def push_imgurl(self, title, url):
 +        try:
 +            self.db.insert({'_id': title, 'statu': self.OUTSTANDING, 'url': url})
 +            print('圖片地址插入成功')
 +        except errors.DuplicateKeyError as e:
 +            print('地址已經存在了')
 +            pass
 +
 +    def pop(self):
 +        """
 +        這個函數會查詢隊列中的所有狀態為OUTSTANDING的值，
 +        更改狀態，（query后面是查詢）（update后面是更新）
 +        并返回_id（就是我們的ＵＲＬ），MongDB好使吧，^_^
 +        如果沒有OUTSTANDING的值則調用repair()函數重置所有超時的狀態為OUTSTANDING，
 +        $set是設置的意思，和MySQL的set語法一個意思
 +        """
 +        record = self.db.find_and_modify(
 +            query={'status': self.OUTSTANDING},
 +            update={'$set': {'status': self.PROCESSING, 'timestamp': datetime.now()}}
 +        )
 +        if record:
 +            return record['_id']
 +        else:
 +            self.repair()
 +            raise KeyError
 +
 +    def pop_title(self, url):
 +        record = self.db.find_one({'_id': url})
 +        return record['主題']
 +
 +    def peek(self):
 +        """這個函數是取出狀態為 OUTSTANDING的文檔并返回_id(URL)"""
 +        record = self.db.find_one({'status': self.OUTSTANDING})
 +        if record:
 +            return record['_id']
 +
 +    def complete(self, url):
 +        """這個函數是更新已完成的URL完成"""
 +        self.db.update({'_id': url}, {'$set': {'status': self.COMPLETE}})
 +
 +    def repair(self):
 +        """這個函數是重置狀態$lt是比較"""
 +        record = self.db.find_and_modify(
 +           query={
 +               'timestamp': {'$lt': datetime.now() - timedelta(seconds=self.timeout)},
 +               'status': {'$ne': self.COMPLETE}
 +           },
 +            update={'$set': {'status': self.OUTSTANDING}}
 +        )
 +        if record:
            print('重置URL狀態', record['_id'])
 

獲取主題頁面all_theme_urls的代碼

from ip_request import html_request
from mongodb_queue import MogoQueue
from bs4 import BeautifulSoup



spider_queue = MogoQueue('meinvxiezhen','crawl_queue')
def start(url):
    response = html_request.get(url,3)
    soup = BeautifulSoup(response,'lxml')
    all_data = soup.find('div', {'class': 'all'}).findAll('a')
    for data in all_data:
        title = data.get_text()
        url = data ['href']
        spider_queue.push(url,title)
if __name__ == '__main__':
    start('http://www.mzitu.com/all')

這里是多線程多進程代碼

import os
import time
import threading
import multiprocessing
from mongodb_queue import MogoQueue
from ip_request import html_request
from download import download_request
from bs4 import BeautifulSoup
sleep_time=2
def meizi_crawler(max_threads = 10):
    crawl_queue = MogoQueue('meinvxiezhen', 'crawl_queue')
    img_queue = MogoQueue('meinvxiezhen', 'img_queue')  ##這個是圖片實際URL的隊列
    def pageurl_crawler():
        while True:
            try :
                url=crawl_queue.pop()
                print(url)
            except KeyError:
                print('隊列沒有數據耶，你好壞耶')
            else:
                img_urls=[]
                title = crawl_queue.pop_title(url)
                path = str(title).replace('?', '')
                mkdir(path)
                os.chdir('C:\\Users\\admin\\Desktop\\mzitu\\{}'.format(path))
                req= html_request.get(url,3)
                max_page = BeautifulSoup(req, 'lxml').find('div', class_='pagenavi').find_all('span')[-2].get_text()
                for page in range(1, int(max_page) + 1):  # 找出每個套圖里面最大的頁數，這里真的不得不佩服我曹哥的想法

                    link = url + '/' + str(page)

                    data = html_request.get(link,3)

                    img_link = BeautifulSoup(data,'lxml').find('div', class_='main-image').find('img')['src']
                    img_urls.append(img_link)
                    download(img_link)
                crawl_queue.complete(url)
                img_queue.push_imgurl(title,img_urls)

    def download(url):#針對每個圖片地址的下載函數
        f=open(url[-9:-4]+'.jpg','ab')
        #必須以二進制寫進去
        f.write(download_request.get(url,3))
        f.close()
    def mkdir(path):#創建文件函數
        isExist = os.path.exists(os.path.join('C:\\Users\\admin\\Desktop\\mzitu',path))#檢驗這個目錄下，path這個文件夾存在嗎，
        #不存在就創建
        if not isExist:
            print('創建一個{}的文件'.format(path))
            os.makedirs(os.path.join('C:\\Users\\admin\\Desktop\\mzitu',path))
            return True
        else:
            print('文件{}已經存在'.format(path))
            return  False
    threads =[]
    while threads or crawl_queue:


#這兒crawl_queue用上了，就是我們__bool__函數的作用，為真則代表我們MongoDB隊列里面還有數據
#threads 或者 crawl_queue為真都代表我們還沒下載完成，程序就會繼續執行

        for thread in threads:
            if not thread.is_alive():
                threads.remove(thread)
        while len(threads)< max_threads or crawl_queue.peek():
            thread = threading.Thread(target=pageurl_crawler())
            thread.setDaemon(True)
            thread.start()
            threads.append(thread)
        time.sleep(sleep_time)
def process_crawler():
    process=[]
    num_cpus=multiprocessing.cpu_count()
    print('將會啟動的進程數為：',num_cpus)
    for i in range (num_cpus):
        p = multiprocessing.Process(target=meizi_crawler)
        p.start()
        process.append(p)  ##添加進進程隊列
    for p in process:
        p.join()  ##等待進程隊列里面的進程結束
if __name__ == '__main__':
    process_crawler()

15分鐘爬了兩百套左右，比單進程單線程快很多

Paste_Image.png