相信大家通過之前的做法都已經搭建起了一個hadoop的開發環境。今天帶大家通過java api來訪問hdfs文件系統

首先啟動hadoop集群

start-dfs.sh
或者
start-all.sh  //一鍵啟動hadoop集群和yarn集群

打開idea

在pom.xml文件里加入hadoop的依賴，我這里使用的是我搭建的一樣版本的依賴
hadoop 2.7.3

<properties>
    <hadoop.version>2.7.3</hadoop.version>
</properties>

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>${hadoop.version}</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>  
    <version>${hadoop.version}</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>${hadoop.version}</version>
</dependency>

Java api在hdfs上創建一個文件目錄

//創建配置文件
Configuration conf = new Configuration();
 //windows下無法找到對應的環境變量，需要設置。把hadoop解壓下來的根目錄
System.setProperty("hadoop.home.dir", "F:\\linux\\hadoop-2.7.3");
// 指定hadoop fs的地址
conf.set("fs.defaultFS", "hdfs://master:9000");
//定義訪問的根目錄
String userRootPath = "/userSpace"
//拿到文件操作對象
FileSystem fs = FileSystem.get(conf);
//創建一個path對象，hdfs上的目錄都需要用path對象來訪問
Path dir = new Path(userRootPath);  //表示根目錄下的 userSpace目錄
boolean result = fs.mkdirs(dir);
if(result){
    System.out.println("創建目錄成功！");
}

其實就和java訪問文件一樣的操作類似，非常簡單，不過操作hdfs主要是通過FileSystem類。下面給大家貼一下完整代碼

package com.mmcc.springboothadoop.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.ByteArrayOutputStream;
import java.io.IOException;

public class HdfsUtils {
    private static Configuration conf = new Configuration();
    public static String userRootPath = "/userSpace";

    static {
        //windows下無法找到對應的環境變量，需要設置
        System.setProperty("hadoop.home.dir", "F:\\linux\\hadoop-2.7.3");
        // 指定hadoop fs的地址
        conf.set("fs.defaultFS", "hdfs://master:9000");
    }

    //判斷路徑是否存在
    public static boolean exists(String path) throws IOException {
        FileSystem fileSystem = FileSystem.get(conf);
        return fileSystem.exists(new Path(path));
    }

    //創建文件
    public static void createFile(String filePath, byte[] contents) throws IOException {
        FileSystem fileSystem = FileSystem.get(conf);
        Path path = new Path(filePath);
        FSDataOutputStream fdo = fileSystem.create(path);
        fdo.write(contents);
        fdo.close();
        fileSystem.close();

    }

    /**
     * 創建文件
     *
     * @param filePath
     * @param fileContent
     * @throws IOException
     */
    public static void createFile(String filePath, String fileContent)
            throws IOException {
        createFile(filePath, fileContent.getBytes());
    }

    //從本地復制到hdfs上
    public static void copyFromLocalFile(String localFilePath, String remoteFilePath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path localPath = new Path(localFilePath);
        Path remotePath = new Path(remoteFilePath);
        fs.copyFromLocalFile(false, true, localPath, remotePath);
        fs.close();
    }


    /**
     * 刪除目錄或文件
     *
     * @param remoteFilePath
     * @param recursive
     * @return
     * @throws IOException
     */
    public static boolean deleteFile(String remoteFilePath, boolean recursive)
            throws IOException {
        FileSystem fs = FileSystem.get(conf);
        boolean result = fs.delete(new Path(remoteFilePath), recursive);
        fs.close();
        return result;
    }
    /**
     * 刪除目錄或文件(如果有子目錄,則級聯刪除)
     *
     * @param remoteFilePath
     * @return
     * @throws IOException
     */
    public static boolean deleteFile(String remoteFilePath) throws IOException {
        return deleteFile(remoteFilePath, true);
    }
    /**
     * 文件重命名
     *
     * @param oldFileName
     * @param newFileName
     * @return
     * @throws IOException
     */
    public static boolean renameFile(String oldFileName, String newFileName)
            throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path oldPath = new Path(oldFileName);
        Path newPath = new Path(newFileName);
        boolean result = fs.rename(oldPath, newPath);
        fs.close();
        return result;
    }
    /**
     * 創建目錄
     *
     * @param dirName
     * @return
     * @throws IOException
     */
    public static boolean createDirectory(String dirName) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path dir = new Path(dirName);
        boolean result = false;
        if (!fs.exists(dir)) {
            result = fs.mkdirs(dir);
        }
        fs.close();
        return result;
    }

    //列出指定路徑下的文件
    public static  RemoteIterator<LocatedFileStatus> listFiles(String dirPath,boolean recursive) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path(dirPath), recursive);//不進行遞歸
        fs.close();
        return listFiles;
    }
    /**
     * 列出指定路徑下的文件（非遞歸）
     *
     * @param basePath
     * @return
     * @throws IOException
     */
    public static RemoteIterator<LocatedFileStatus> listFiles(String basePath)
            throws IOException {
        FileSystem fs = FileSystem.get(conf);
        RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(
                new Path(basePath), false);
        fs.close();
        return remoteIterator;
    }

    /**
     * 列出指定目錄下的文件\子目錄信息（非遞歸）
     *
     * @param dirPath
     * @return
     * @throws IOException
     */
    public static FileStatus[] listStatus(String dirPath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] fileStatuses = fs.listStatus(new Path(dirPath));
        fs.close();
        return fileStatuses;
    }

    //讀取文件內容
    public static byte[] readFile(String filePath) throws IOException {
        byte[] fileContent = null;
        FileSystem fs = FileSystem.get(conf);
        Path path = new Path(filePath);

        if (fs.exists(path)){
            FSDataInputStream fsin = fs.open(path);
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            IOUtils.copyBytes(fsin,bos,conf);
            fileContent = bos.toByteArray();
        }
        return fileContent;
    }

    //下載hdfs上的文件
    public static void download(String remote,String local) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        //遠程hdfs上的文件
        Path remotePath = new Path(remote);
        //本地的文件
        Path localPath = new Path(local);
        fs.copyToLocalFile(remotePath,localPath);
        fs.close();
    }
}