通过IO流操作 HDFS代码实现-CFANZ编程社区

HDFS文件上传

@Test
      /**
       * HDFS文件上传
       */
      public void putFileToHDFS() throws URISyntaxException, IOException, InterruptedException {
          //1.获取文件系统
          Configuration configuration = new Configuration();
          FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
          //2.创建输入流
          FileInputStream fis = new FileInputStream(new File("e:/hostsmap.txt"));
          //3.获取输出流
          FSDataOutputStream fos = fs.create(new Path("/114.txt"));
          //4.流对拷
          IOUtils.copyBytes(fis,fos,configuration);

          //5.关闭资源
          IOUtils.closeStream(fos);
          IOUtils.closeStream(fis);
          fs.close();

      }

HDFS文件下载

/**
         * HDFS文件的下载
         */
        @Test
        public void getFileFromHDFS() throws URISyntaxException, IOException, InterruptedException {
            //1 获取文件系统
            Configuration configuration = new Configuration();
            FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
            //2 获取输入流
            FSDataInputStream fis = fs.open(new Path("/114.txt"));

            //3 获取输出流
            FileOutputStream fos = new FileOutputStream(new File("e:/wo114.txt"));
            //4 流的对拷
            IOUtils.copyBytes(fis,fos,configuration);
            //5 关闭资源
            IOUtils.closeStream(fos);
            IOUtils.closeStream(fis);
            fs.close();
    }

HDFS文件的定位某一块进行下载

下载第一块

/**
     * 下载第一块
     */
    @Test
    public void readFileSeek() throws InterruptedException, IOException, URISyntaxException {
        //1 获取文件系统
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
        //2 获取输入流
        FSDataInputStream fis = fs.open(new Path("/user/hadoop-2.7.2.tar.gz"));
        //3 创建输出流
        FileOutputStream fos = new FileOutputStream(new File("e:/hadoop-2.7.2.tar.gz.part1"));
        //4 流的拷贝
        byte[] buf = new byte[1024];

        for (int i = 0; i < 1024*128; i++) {
            fis.read(buf);
            fos.write(buf);
        }
        //5 关闭资源
        IOUtils.closeStream(fos);
        IOUtils.closeStream(fis);
    }

下载第二块

/**
     * 下载第二块
     */
    @Test
    public void readFileSeek2() throws URISyntaxException, IOException, InterruptedException {
        //1 获取文件系统
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
        //2 打开输入流
        FSDataInputStream fis = fs.open(new Path("/user/hadoop-2.7.2.tar.gz"));
        //3 定位输入数据位置
        fis.seek(1024*1024*128);
        //4 创建输出流
        FileOutputStream fos = new FileOutputStream(new File("e:/hadoop-2.7.2.tar.gz.part2"));
        //5 流的对拷
        IOUtils.copyBytes(fis,fos,configuration);
        //6 关闭资源
        IOUtils.closeStream(fos);
        IOUtils.closeStream(fis);

    }