相信大家通過(guò)之前的做法都已經(jīng)搭建起了一個(gè)hadoop的開(kāi)發(fā)環(huán)境。今天帶大家通過(guò)java api來(lái)訪問(wèn)hdfs文件系統(tǒng)
首先啟動(dòng)hadoop集群
start-dfs.sh
或者
start-all.sh //一鍵啟動(dòng)hadoop集群和yarn集群
打開(kāi)idea
在pom.xml文件里加入hadoop的依賴，我這里使用的是我搭建的一樣版本的依賴
hadoop 2.7.3
<properties>
<hadoop.version>2.7.3</hadoop.version>
</properties>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>
Java api在hdfs上創(chuàng)建一個(gè)文件目錄
//創(chuàng)建配置文件
Configuration conf = new Configuration();
//windows下無(wú)法找到對(duì)應(yīng)的環(huán)境變量，需要設(shè)置。把hadoop解壓下來(lái)的根目錄
System.setProperty("hadoop.home.dir", "F:\\linux\\hadoop-2.7.3");
// 指定hadoop fs的地址
conf.set("fs.defaultFS", "hdfs://master:9000");
//定義訪問(wèn)的根目錄
String userRootPath = "/userSpace"
//拿到文件操作對(duì)象
FileSystem fs = FileSystem.get(conf);
//創(chuàng)建一個(gè)path對(duì)象，hdfs上的目錄都需要用path對(duì)象來(lái)訪問(wèn)
Path dir = new Path(userRootPath); //表示根目錄下的 userSpace目錄
boolean result = fs.mkdirs(dir);
if(result){
System.out.println("創(chuàng)建目錄成功！");
}
其實(shí)就和java訪問(wèn)文件一樣的操作類(lèi)似，非常簡(jiǎn)單，不過(guò)操作hdfs主要是通過(guò)FileSystem類(lèi)。下面給大家貼一下完整代碼
package com.mmcc.springboothadoop.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
public class HdfsUtils {
private static Configuration conf = new Configuration();
public static String userRootPath = "/userSpace";
static {
//windows下無(wú)法找到對(duì)應(yīng)的環(huán)境變量，需要設(shè)置
System.setProperty("hadoop.home.dir", "F:\\linux\\hadoop-2.7.3");
// 指定hadoop fs的地址
conf.set("fs.defaultFS", "hdfs://master:9000");
}
//判斷路徑是否存在
public static boolean exists(String path) throws IOException {
FileSystem fileSystem = FileSystem.get(conf);
return fileSystem.exists(new Path(path));
}
//創(chuàng)建文件
public static void createFile(String filePath, byte[] contents) throws IOException {
FileSystem fileSystem = FileSystem.get(conf);
Path path = new Path(filePath);
FSDataOutputStream fdo = fileSystem.create(path);
fdo.write(contents);
fdo.close();
fileSystem.close();
}
/**
* 創(chuàng)建文件
*
* @param filePath
* @param fileContent
* @throws IOException
*/
public static void createFile(String filePath, String fileContent)
throws IOException {
createFile(filePath, fileContent.getBytes());
}
//從本地復(fù)制到hdfs上
public static void copyFromLocalFile(String localFilePath, String remoteFilePath) throws IOException {
FileSystem fs = FileSystem.get(conf);
Path localPath = new Path(localFilePath);
Path remotePath = new Path(remoteFilePath);
fs.copyFromLocalFile(false, true, localPath, remotePath);
fs.close();
}
/**
* 刪除目錄或文件
*
* @param remoteFilePath
* @param recursive
* @return
* @throws IOException
*/
public static boolean deleteFile(String remoteFilePath, boolean recursive)
throws IOException {
FileSystem fs = FileSystem.get(conf);
boolean result = fs.delete(new Path(remoteFilePath), recursive);
fs.close();
return result;
}
/**
* 刪除目錄或文件(如果有子目錄,則級(jí)聯(lián)刪除)
*
* @param remoteFilePath
* @return
* @throws IOException
*/
public static boolean deleteFile(String remoteFilePath) throws IOException {
return deleteFile(remoteFilePath, true);
}
/**
* 文件重命名
*
* @param oldFileName
* @param newFileName
* @return
* @throws IOException
*/
public static boolean renameFile(String oldFileName, String newFileName)
throws IOException {
FileSystem fs = FileSystem.get(conf);
Path oldPath = new Path(oldFileName);
Path newPath = new Path(newFileName);
boolean result = fs.rename(oldPath, newPath);
fs.close();
return result;
}
/**
* 創(chuàng)建目錄
*
* @param dirName
* @return
* @throws IOException
*/
public static boolean createDirectory(String dirName) throws IOException {
FileSystem fs = FileSystem.get(conf);
Path dir = new Path(dirName);
boolean result = false;
if (!fs.exists(dir)) {
result = fs.mkdirs(dir);
}
fs.close();
return result;
}
//列出指定路徑下的文件
public static RemoteIterator<LocatedFileStatus> listFiles(String dirPath,boolean recursive) throws IOException {
FileSystem fs = FileSystem.get(conf);
RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path(dirPath), recursive);//不進(jìn)行遞歸
fs.close();
return listFiles;
}
/**
* 列出指定路徑下的文件（非遞歸）
*
* @param basePath
* @return
* @throws IOException
*/
public static RemoteIterator<LocatedFileStatus> listFiles(String basePath)
throws IOException {
FileSystem fs = FileSystem.get(conf);
RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(
new Path(basePath), false);
fs.close();
return remoteIterator;
}
/**
* 列出指定目錄下的文件\子目錄信息（非遞歸）
*
* @param dirPath
* @return
* @throws IOException
*/
public static FileStatus[] listStatus(String dirPath) throws IOException {
FileSystem fs = FileSystem.get(conf);
FileStatus[] fileStatuses = fs.listStatus(new Path(dirPath));
fs.close();
return fileStatuses;
}
//讀取文件內(nèi)容
public static byte[] readFile(String filePath) throws IOException {
byte[] fileContent = null;
FileSystem fs = FileSystem.get(conf);
Path path = new Path(filePath);
if (fs.exists(path)){
FSDataInputStream fsin = fs.open(path);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
IOUtils.copyBytes(fsin,bos,conf);
fileContent = bos.toByteArray();
}
return fileContent;
}
//下載hdfs上的文件
public static void download(String remote,String local) throws IOException {
FileSystem fs = FileSystem.get(conf);
//遠(yuǎn)程hdfs上的文件
Path remotePath = new Path(remote);
//本地的文件
Path localPath = new Path(local);
fs.copyToLocalFile(remotePath,localPath);
fs.close();
}
}
