想爬豆瓣相冊起因是因為，想將豆瓣照片轉移到其他地方，但是豆瓣的照片下載下來都是webp格式，正常情況下沒法打開，所以想到能不能通過selenium爬到數據呢？
E:\IDEAProject\MyProject\SeleniumTest

本來想的是從模擬登陸開始，然后一步步進到相冊里，獲取相冊圖片列表，但是比較麻煩。還有一個更方便的操作，通過F12開發者工具，查看請求的url，帶上cookie去模擬請求

進到相冊首頁

GET請求帶cookie，https://www.douban.com/people/62414040/photos

image.png

通過對Response的分析

image.png

看到相冊信息以及相冊地址都在下圖所示的信息塊中
通過對該信息塊進行信息提取，可以得到相冊名列表和相冊地址url列表

image.png

進到相冊詳情頁

GET請求帶cookie，https://www.douban.com/photos/album/1871536872/
如果相冊里圖片數量超過了18張，就會有第二頁
GET請求帶cookie，https://www.douban.com/photos/album/1871536872/?m_start=18
同理，如果有第三頁第四頁，url是
https://www.douban.com/photos/album/1871536872/?m_start=36
https://www.douban.com/photos/album/1871536872/?m_start=54

image.png

通過對Response的分析
看到相冊信息以及相冊中每張圖片url都在下圖所示的信息塊中

image.png

有了圖片地址就好辦了，使用IO流下載圖片

代碼如下

獲取每個相冊中圖片的地址，返回集合

    /**
     * GET方式
     * 獲取每個相冊中圖片的地址，返回集合
     */
    public List<String> getImageUrlList(String passUrl, String folderName) throws BusinessException {

        //第五步：在獲取以上憑證后開始采集數據
        logger.info("獲取豆瓣相冊" + folderName + "圖片地址");

//        String passUrl = "https://www.douban.com/photos/album/1871536796/";
        String passCookie = "gmxq-IDUYXw; douban-fav-remind=1; __yadk_uid=bK748gKAh8REVU6PsYrAF24ZroyVxSYA; " +
                "__gads=ID=9cf216a578728a98:T=1587890575:S=ALNI_MbpwRrOxhrMiUZPlDmEcg-YeTzuhw; ll=\"118159\"; " +
                "__utmc=30149280; ap_v=0,6.0; push_noty_num=0; push_doumail_num=0; __utmv=30149280.6241; " +
                "douban-profile-remind=1; ct=y; __utmz=30149280.1588515963.4.3.utmcsr=baidu|utmccn=(organic)" +
                "|utmcmd=organic|utmctr=%E8%B1%86%E7%93%A3%E7%9B%B8%E5%86%8C%E4%B8%8B%E8%BD%BD%E6%80%8E%E4%B9%88%E6%98" +
                "%AFwebp; dbcl2=\"62414040:cC/OMBA010s\"; ck=snAu; gr_user_id=7f5f3e5d-9cf9-4d38-9e3e-8ffebe3b22b1; " +
                "_pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1588520368%2C%22https%3A%2F%2Fwww.baidu" +
                ".com%2Fbaidu%3Fisource%3Dinfinity%26iname%3Dbaidu%26itype%3Dweb%26tn%3D02003390_42_hao_pg%26ie%3Dutf-8" +
                "%26wd%3D5kg%25E5%25A4%25A7%25E7%25B1%25B3%25E4%25B8%2580%25E4%25B8%25AA%25E4%25BA%25BA%25E5%2590%2583" +
                "%25E5%25A4%259A%25E4%25B9%2585%22%5D; _pk_ses.100001.8cb4=*; __utma=30149280.835562099.1587890488" +
                ".1588515963.1588520368.5; __utmt=1; _pk_id.100001.8cb4=d2bb86d9f8460bdf.1587890487.3.1588520483" +
                ".1588517154.; __utmb=30149280.12.10.1588520368";

        List<String> imageUrl = new ArrayList<>();

        //請求url帶上cookie
        Document document = RequestUtil.sendGetRequestWithNullReturnDoc(passUrl, passCookie);
        //圖片元素列表
        Elements photo_wrap = document.getElementsByClass("photo_wrap");
        //圖片元素列表for循環，單個元素有哪些子元素，通過getElementsByTag方法得到子元素后就，再調用attr方法獲取屬性值
        for (Element element : photo_wrap) {

            Elements allElements = element.getAllElements();
            Elements img = element.getElementsByTag("img");
            String src = img.attr("src");
            System.out.println(src);

            //將webp替換成jpg，一樣可以打開圖片
            src = src.replace("webp", "jpg");
            imageUrl.add(src);

        }

        return imageUrl;
    }

下載圖片方法

    /**
     * java 通過url下載圖片保存到本地
     *
     * @param urlString 圖片url地址
     * @param i         圖片保存名稱，雪花算法【需要引入mybatis-plus】
     * @throws Exception
     */
    public static void download(String urlString, String basePath, String folderName, Long i) throws Exception {

        // 構造URL
        URL url = new URL(urlString);
        // 打開連接
        URLConnection con = url.openConnection();
        // 輸入流
        InputStream is = con.getInputStream();
        // 1K的數據緩沖
        byte[] bs = new byte[1024];
        // 讀取到的數據長度
        int len;

        //完整文件夾名
        File file0 = new File(basePath + "\\" + folderName);
        if (!file0.isDirectory() && !file0.exists()) {
            file0.mkdirs();
        }

        // 輸出的文件流
        String filename = file0 + "\\" + i + ".jpg";  //下載路徑及下載圖片名稱

        File file = new File(filename);
        FileOutputStream os = new FileOutputStream(file, true);
        // 開始讀取
        while ((len = is.read(bs)) != -1) {
            os.write(bs, 0, len);
        }
        System.out.println(i);
        // 完畢，關閉所有鏈接
        os.close();
        is.close();
    }

    }
}

獲取相冊的集合

    /**
     * 獲取相冊的集合
     */
    public List<List<String>> getAlbumList(String passUrl) throws BusinessException {
        logger.info("圖片首頁");
        String passCookie = "gmxq-IDUYXw; douban-fav-remind=1; __yadk_uid=bK748gKAh8REVU6PsYrAF24ZroyVxSYA; " +
                "__gads=ID=9cf216a578728a98:T=1587890575:S=ALNI_MbpwRrOxhrMiUZPlDmEcg-YeTzuhw; ll=\"118159\"; " +
                "__utmc=30149280; ap_v=0,6.0; push_noty_num=0; push_doumail_num=0; __utmv=30149280.6241; " +
                "douban-profile-remind=1; ct=y; __utmz=30149280.1588515963.4.3.utmcsr=baidu|utmccn=(organic)" +
                "|utmcmd=organic|utmctr=%E8%B1%86%E7%93%A3%E7%9B%B8%E5%86%8C%E4%B8%8B%E8%BD%BD%E6%80%8E%E4%B9%88%E6%98" +
                "%AFwebp; dbcl2=\"62414040:cC/OMBA010s\"; ck=snAu; gr_user_id=7f5f3e5d-9cf9-4d38-9e3e-8ffebe3b22b1; " +
                "_pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1588520368%2C%22https%3A%2F%2Fwww.baidu" +
                ".com%2Fbaidu%3Fisource%3Dinfinity%26iname%3Dbaidu%26itype%3Dweb%26tn%3D02003390_42_hao_pg%26ie%3Dutf-8" +
                "%26wd%3D5kg%25E5%25A4%25A7%25E7%25B1%25B3%25E4%25B8%2580%25E4%25B8%25AA%25E4%25BA%25BA%25E5%2590%2583" +
                "%25E5%25A4%259A%25E4%25B9%2585%22%5D; _pk_ses.100001.8cb4=*; __utma=30149280.835562099.1587890488" +
                ".1588515963.1588520368.5; __utmt=1; _pk_id.100001.8cb4=d2bb86d9f8460bdf.1587890487.3.1588520483" +
                ".1588517154.; __utmb=30149280.12.10.1588520368";

        //相冊的集合
        List<List<String>> albumListAll = new ArrayList<>();

        //請求url帶上cookie
        Document document = RequestUtil.sendGetRequestWithNullReturnDoc(passUrl, passCookie);
        Elements allElements = document.getAllElements();
        Elements albumlst = document.getElementsByClass("albumlst");

        for (Element element : albumlst) {

            List<String> albumList = new ArrayList<>();

            Elements elements = element.getElementsByClass("albumlst");
            Element element1 = elements.get(0);

            String href = element1.getElementsByClass("album_photo").attr("href");
            String pl = element1.getElementsByClass("pl").text();
            String substring = pl.substring(0, pl.indexOf("張"));
            int totalRecord = Integer.parseInt(substring);

            //totalRecord:總記錄數 / pageSize:每頁多少條記錄  /totalPageNum:總頁數
            int pageSize = 18;
            //分頁的總頁數算法
            int totalPageNum = (totalRecord + pageSize - 1) / pageSize;

            if (totalPageNum == 1) {
                albumList.add(href);
            } else if (totalPageNum >= 2) {
                albumList.add(href);
                for (int i = 1; i < totalPageNum; i++) {
                    String s = href + "?m_start=" + pageSize * i;
                    albumList.add(s);
                }
            } else {

            }
            albumListAll.add(albumList);
        }
        return albumListAll;
    }

獲取相冊名的集合

    /**
     * 獲取相冊名的集合
     */
    public List<String> getAlbumNameList(String passUrl) throws BusinessException {
        logger.info("圖片首頁");
        String passCookie = "gmxq-IDUYXw; douban-fav-remind=1; __yadk_uid=bK748gKAh8REVU6PsYrAF24ZroyVxSYA; " +
                "__gads=ID=9cf216a578728a98:T=1587890575:S=ALNI_MbpwRrOxhrMiUZPlDmEcg-YeTzuhw; ll=\"118159\"; " +
                "__utmc=30149280; ap_v=0,6.0; push_noty_num=0; push_doumail_num=0; __utmv=30149280.6241; " +
                "douban-profile-remind=1; ct=y; __utmz=30149280.1588515963.4.3.utmcsr=baidu|utmccn=(organic)" +
                "|utmcmd=organic|utmctr=%E8%B1%86%E7%93%A3%E7%9B%B8%E5%86%8C%E4%B8%8B%E8%BD%BD%E6%80%8E%E4%B9%88%E6%98" +
                "%AFwebp; dbcl2=\"62414040:cC/OMBA010s\"; ck=snAu; gr_user_id=7f5f3e5d-9cf9-4d38-9e3e-8ffebe3b22b1; " +
                "_pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1588520368%2C%22https%3A%2F%2Fwww.baidu" +
                ".com%2Fbaidu%3Fisource%3Dinfinity%26iname%3Dbaidu%26itype%3Dweb%26tn%3D02003390_42_hao_pg%26ie%3Dutf-8" +
                "%26wd%3D5kg%25E5%25A4%25A7%25E7%25B1%25B3%25E4%25B8%2580%25E4%25B8%25AA%25E4%25BA%25BA%25E5%2590%2583" +
                "%25E5%25A4%259A%25E4%25B9%2585%22%5D; _pk_ses.100001.8cb4=*; __utma=30149280.835562099.1587890488" +
                ".1588515963.1588520368.5; __utmt=1; _pk_id.100001.8cb4=d2bb86d9f8460bdf.1587890487.3.1588520483" +
                ".1588517154.; __utmb=30149280.12.10.1588520368";

        //相冊的集合
        List<String> albumNameList = new ArrayList<>();

        //請求url帶上cookie
        Document document = RequestUtil.sendGetRequestWithNullReturnDoc(passUrl, passCookie);
        Elements allElements = document.getAllElements();
        Elements albumlst = document.getElementsByClass("albumlst");
        for (Element element : albumlst) {

            Elements elements = element.getElementsByClass("albumlst");
            Element element1 = elements.get(0);

            Elements elements1 = element1.getElementsByClass("pl2");
            String albumName = elements1.text();

            albumNameList.add(albumName);
        }


        return albumNameList;
    }