关于HDFS的几个操作纯代码

一.功能实现：

1：将本地文件上传到HDFS上

2：从HDFS上读取文件到本地

3：删除HDFS上的文件

4：遍历HDFS上的文件

注：

1：导入google-collections-xx.jar文件。

2：文件目标地址要具体到文件名，而不能是文件夹。

二.代码实现：

package com.hdfs;

importJava.io.BufferedInputStream;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;

import java.NET.URI;

import org.apache.Hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IOUtils;

import org.apache.hadoop.util.Progressable;

public class HdfsOperation {

/**

* 上传文件到HDFS上去

*/

private static void uploadToHdfs() throws FileNotFoundException,

IOException {

String localSrc = "/home/file2/a.txt";

String dst = "hdfs://master:9000/user/root/input0/a.txt";

InputStream in = new BufferedInputStream(new FileInputStream(localSrc));

Configuration conf = new Configuration();

FileSystem fs = FileSystem.get(URI.create(dst), conf);

OutputStream out = fs.create(new Path(dst), new Progressable() {

public void progress() {

System.out.print(".");

}

});

IOUtils.copyBytes(in, out, 4096, true);

}

/**

* 从HDFS上读取文件

*/

private static void readFromHdfs() throws FileNotFoundException,IOException {

String dst = "hdfs://192.10.5.76:9000/user/root/input0/a.txt";

Configuration conf = new Configuration();

FileSystem fs = FileSystem.get(URI.create(dst), conf);

FSDataInputStream hdfsInStream = fs.open(new Path(dst));

OutputStream out = new FileOutputStream("/home/li");

byte[] ioBuffer = new byte[1024];

int readLen = hdfsInStream.read(ioBuffer);

while(-1 != readLen){

out.write(ioBuffer, 0, readLen);

readLen = hdfsInStream.read(ioBuffer);

}

out.close();

hdfsInStream.close();

fs.close();

}

/**

* HDFS上删除文件

* @throws FileNotFoundException

* @throws IOException

*/

private static void deleteFromHdfs() throws FileNotFoundException,IOException {

String dst = "hdfs://master:9000/user/root/input0/a.txt";

Configuration conf = new Configuration();

FileSystem fs = FileSystem.get(URI.create(dst), conf);

fs.deleteOnExit(new Path(dst));

fs.close();

}

/**

* 遍历HDFS上的文件和目录

*/

private static void getDirectoryFromHdfs() throws FileNotFoundException,IOException {

String dst = "hdfs://master:9000/user/root/input0";

Configuration conf = new Configuration();

FileSystem fs = FileSystem.get(URI.create(dst), conf);

FileStatus fileList[] = fs.listStatus(new Path(dst));

int size = fileList.length;

for(int i = 0; i < size; i++){

System.out.println("name:" + fileList[i].getPath().getName() + "\t\tsize:" + fileList[i].getLen());

}

fs.close();

}

/**

* main函数

* @param args

* @throws Exception

*/

public static void main(String[] args) throws Exception {

try {

uploadToHdfs();

readFromHdfs();

deleteFromHdfs();

getDirectoryFromHdfs();

} catch (Exception e) {

// TODO Auto-generated catch block

System.out.println(2);

e.printStackTrace();

}

finally {

System.out.println("SUCCESS");

System.out.println(3);

}

}

}