Maven

通俗上讲就是我们不在手动导入jar包，交给Maven去管理
Maven通过pom.xml中的坐标进行管理

• 新建Maven项目
  
  
  导入依赖

  <url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
</properties>
<repositories>
<repository>
    <id>cloudera</id>
    <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>

<dependencies>
<!--        添加hadoop依赖-->
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>${hadoop.version}</version>
</dependency>

<!--        添加junit依赖-->
<dependency>
    <groupId>junit</groupId>
    <artifactId>junit</artifactId>
    <version>4.10</version>
    <scope>test</scope>
</dependency>


</dependencies>

使用Java api在hdfs上新建文件夹

package hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
public class HDFSAPP {
    Configuration configuration = null;
    FileSystem fileSystem = null;
    public static final String HDFS_PATH = "hdfs://192.168.2.4/:8020";
    @Test
    public void mkdir() throws Exception {
         fileSystem.mkdirs(new Path("/ideahehe/test"));

    }
    // 测试之前要执行的代码

    @Before
    public void setUp() throws Exception {
        System.out.println("开始建立与HDFS的连接");
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration,"hadoop");
    }
    // 测试之后要执行的代码
    @After
    public void tearDown(){


        configuration = null;
        fileSystem =  null;
        System.out.println("关闭与HDFS的连接");
    }

}

// 创建文件

  
    @Test
    public void create() throws Exception {
        Path path = new Path("/ideahehe/test/hello.txt");
        FSDataOutputStream outputStream = fileSystem.create(path);
        outputStream.write("hello taiyuan gongye ".getBytes());
        outputStream.flush();
        outputStream.close();


    }

   // rename文件
    @Test
    public void rename() throws Exception {
        Path oldPath = new Path("/ideahehe/test/hello.txt");
        Path newPath = new Path("/ideahehe/test/xixi.txt");
        fileSystem.rename(oldPath, newPath);
    }

 // 查看文件
    @Test
    public void cat() throws Exception {
        Path newPath = new Path("/ideahehe/test/xixi.txt");
        FSDataInputStream inputStream = fileSystem.open(newPath);
        IOUtils.copyBytes(inputStream, System.out, 1024);
        inputStream.close();

    }

package com.neusoft.hadoop.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * @author Eric Lee
 * @date 2020/10/26 14:45
 * Java 操作HDFS
 */
public class HDFSApp {
    Configuration configuration = null;
    FileSystem fileSystem = null;
    public static final String HDFS_PATH = "hdfs://10.25.187.18:8020";

    /**
     * 在 hdfs中新建 文件夹    /你的名字/test
     * @throws Exception
     */
    @Test
    public void mkdir() throws Exception {
        fileSystem.mkdirs(new Path("/liyan33/test"));
    }

    /**
     * 创建文件
     * @throws Exception
     */

    @Test
    public void create() throws Exception {
        FSDataOutputStream outputStream = fileSystem.create(new Path("/liyan33/test/haha.txt"));
        outputStream.write("hello qilulihgong bigdata student".getBytes());
        outputStream.flush();
        outputStream.close();
    }


    /**
     * 查看文件
     * @throws Exception
     * hdfs -fs -cat path
     */
    @Test
    public void cat() throws Exception {
        FSDataInputStream in = fileSystem.open(new Path("/liyan22/test/hehe.txt"));
        IOUtils.copyBytes(in, System.out, 1024);
        in.close();

    }

    /**
     * 重名文件
     * @throws Exception
     *
     */
    @Test
    public void rename() throws Exception {
        Path oldPath = new Path("/liyan22/test/haha.txt");
        Path newPath = new Path("/liyan22/test/hehe.txt");

        fileSystem.rename(oldPath, newPath);
    }
    /**
     * 上传文件到HDFS
     * @throws Exception
     *
     */
    @Test
    public void copyFromLocalFile() throws Exception {
        Path loacalPath = new Path("hello.txt");
        Path hdfsPath = new Path("/");

        fileSystem.copyFromLocalFile(loacalPath, hdfsPath);
    }


    /**
     * 上传文件到HDFS带进度信息
     * @throws Exception
     *
     */

    @Test
    public void copyFromLocalFileWithProgress() throws Exception {
        InputStream in = new BufferedInputStream(
                new FileInputStream(new File("cifar-10-python.tar.gz"))
        );

        FSDataOutputStream ouput = fileSystem.create(new Path("/liyan33/test/cifar-10-python.tar.gz"),
                (Progressable) () -> {
                    System.out.print(".");
                });
        IOUtils.copyBytes(in, ouput, 4096);
    }
    /**
     * 下载文件到HDFS
     * @throws Exception
     *
     */


    // TODO
    @Test
    public void copyToLocalFile() throws Exception {
//        Path hdfsPath = new Path("/liyan33/test/cifar-10-python.tar.gz");
//        Path loacalPath = new Path("./dl/cifar-10-python.tar.gz");
                Path hdfsPath = new Path("/liyan33/test/haha.txt");
        Path loacalPath = new Path("./dl/haha.txt");
        //  useRawLocalFileSystem
        fileSystem.copyToLocalFile(false, hdfsPath, loacalPath, true);
    }
    /**
     * 查看某个目录下所有文件
     * @throws Exception
     *
     */
    @Test
    public void listFiles() throws Exception{
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        for (FileStatus f :fileStatuses){
         String isDir =  f.isDirectory()? "文件夹" : "文件";
            short replication = f.getReplication();
            long len = f.getLen();
            String path = f.getPath().toString();

            System.out.println(isDir + "\t" + replication +"\t" + len + "\t" +path);

        }
    }

    /**
     * 删除文件
     * @throws Exception
     *
     */
    @Test
    public void  delete() throws IOException {
        fileSystem.delete(new Path("/heyueqian"), true);
    }










    //测试之前执行的代码
    @Before
    public void setUp() throws Exception {
        System.out.println("开始建立与HDFS的连接");
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration, "hadoop");
    }
    //测试之完执行的代码
    @After
    public void tearDown(){
        configuration = null;
        fileSystem = null;
        System.out.println("关闭与HDFS的连接");



    }

}