hadoop – 未来可期

hadoop 平衡节点数据

在集群运行中，会发现在不同的节点中空间利用率是不同的，甚至会造成有的节点数据快存满了，有的利用率还不… 继续阅读 hadoop 平衡节点数据

Hadoop的内存配置

hdp-configuration-utils.py Hadoop的内存配置有两种方法：利用手动安装… 继续阅读 Hadoop的内存配置

maven引入hadoop


&lt;dependency&gt;
  &lt;groupId&gt;org.apache.hadoop&lt;/groupId&gt;
  &lt;artifactId&gt;hadoop-client&lt;/artifactId&gt;
  &lt;version&gt;3.1.3&lt;/version&gt;
&lt;/dependency&gt;

<groupId>org.apache.hadoop</groupId>

<artifactId>hadoop-client</artifactId>

</dependency>

hadoop hdfs 文件操作


package com.imooc.bigdata.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;

public class HDFSApp {
    public static final String HDFS_PATH = "hdfs://192.168.64.131:9000";
    FileSystem fileSystem = null;
    Configuration configuration = null;

    @Before
    public void setUp() throws URISyntaxException, IOException, InterruptedException {
        System.out.println("------------setup----------");
        configuration = new Configuration();
        //设置副本个数
        configuration.set("dfs.replication","1");
        fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration,"root");
    }

    //创建目录
    @Test
    public void mkdir() throws IOException {
        System.out.println("------------mkdir----------");
        fileSystem.mkdirs(new Path("/hdfsapi/test"));
    }
    //创建并写入文件
    @Test
    public void create() throws Exception{
        FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/b.txt"));
//        FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/a.txt"));
        out.writeUTF("hello pk dfs.replication 1");
        out.flush();
        out.close();
    }
    //读取文件
    @Test
    public void text() throws IOException {
        FSDataInputStream FSDataInputStream = fileSystem.open(new Path("/hdfsapi/test/a.txt"));
        IOUtils.copyBytes(FSDataInputStream,System.out,1024);
    }

    //更改文件名
    @Test
    public void rename() throws Exception{
        Path oldPath = new Path("/hdfsapi/test/b.txt");
        Path newPath = new Path("/hdfsapi/test/c.txt");
        boolean res = fileSystem.rename(oldPath,newPath);
        System.out.println(res);
    }

    //拷贝本地文件到hdfs系统
    @Test
    public void copyFromLocalFile() throws Exception{
        Path src = new Path("E:\\download\\count.exe");
        Path dst = new Path("/hdfsapi/test02/");
        fileSystem.copyFromLocalFile(src,dst);
    }

    //拷贝大文件
    @Test
    public void copyFromLocalBigFile() throws Exception{
        File File = new File("C:\\Users\\Administrator\\Desktop\\jdk-8u191-linux-x64.tar.gz");
        InputStream in = new BufferedInputStream(new FileInputStream(File));
        FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/jdk-8u191-linux-x64.tar.gz"), new Progressable() {
            @Override
            public void progress() {
                System.out.print("-");
            }
        });
        IOUtils.copyBytes(in,out,4096);
    }

    //拷贝文件到本地目录
    @Test
    public void copyToLocalFile() throws Exception{
        Path src = new Path("/hdfsapi/test/a.txt");
        Path dst = new Path("e:/");
        fileSystem.copyToLocalFile(src,dst);
    }

    //查看指定文件夹下所有文件
    @Test
    public void listFiles() throws Exception{
        FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi/test"));
        for (FileStatus file:statuses) {
            System.out.println(file.isDirectory() ? "文件夹":"文件" + ":" + file.getPath());;
        }
    }

    //递归查看指定文件夹下所有文件
    @Test
    public void listFilesRecursive() throws Exception{
        RemoteIterator&lt;LocatedFileStatus&gt; files = fileSystem.listFiles(new Path("/hdfsapi/"),true);

        while (files.hasNext()){
            LocatedFileStatus file = files.next();
            System.out.println(file.getPath());
        }
    }
    //查看文件块信息
    @Test
    public void getFileBlockLocations() throws Exception{
        FileStatus status = fileSystem.getFileStatus(new Path("/hdfsapi/test/jdk-8u191-linux-x64.tar.gz"));
        BlockLocation[] locations = fileSystem.getFileBlockLocations(status,0,status.getLen());
        for (BlockLocation location: locations) {
            for (String name:location.getNames()) {
                System.out.println(name + ":" + location.getOffset() + ":" + location.getLength());
            }
        }
    }

    //删除文件
    @Test
    public void delete() throws Exception{
        boolean res = fileSystem.delete(new Path("/hdfsapi/test/a.txt"));
        System.out.println(res);
    }
    @After
    public void tearDown(){
        System.out.println("------------tearDown----------");
        configuration = null;
        fileSystem = null;
    }
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

package com.imooc.bigdata.hdfs;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.*;

import org.apache.hadoop.io.IOUtils;

import org.apache.hadoop.util.Progressable;

import org.junit.After;

import org.junit.Before;

import org.junit.Test;

import java.io.*;

import java.net.URI;

import java.net.URISyntaxException;

public class HDFSApp {

public static final String HDFS_PATH = "hdfs://192.168.64.131:9000";

FileSystem fileSystem = null;

Configuration configuration = null;

@Before

public void setUp() throws URISyntaxException, IOException, InterruptedException {

System.out.println("------------setup----------");

configuration = new Configuration();

//设置副本个数

configuration.set("dfs.replication","1");

fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration,"root");

}

//创建目录

@Test

public void mkdir() throws IOException {

System.out.println("------------mkdir----------");

fileSystem.mkdirs(new Path("/hdfsapi/test"));

}

//创建并写入文件

@Test

public void create() throws Exception{

FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/b.txt"));

// FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/a.txt"));

out.writeUTF("hello pk dfs.replication 1");

out.flush();

out.close();

}

//读取文件

@Test

public void text() throws IOException {

FSDataInputStream FSDataInputStream = fileSystem.open(new Path("/hdfsapi/test/a.txt"));

IOUtils.copyBytes(FSDataInputStream,System.out,1024);

}

//更改文件名

@Test

public void rename() throws Exception{

Path oldPath = new Path("/hdfsapi/test/b.txt");

Path newPath = new Path("/hdfsapi/test/c.txt");

boolean res = fileSystem.rename(oldPath,newPath);

System.out.println(res);

}

//拷贝本地文件到hdfs系统

@Test

public void copyFromLocalFile() throws Exception{

Path src = new Path("E:\\download\\count.exe");

Path dst = new Path("/hdfsapi/test02/");

fileSystem.copyFromLocalFile(src,dst);

}

//拷贝大文件

@Test

public void copyFromLocalBigFile() throws Exception{

File File = new File("C:\\Users\\Administrator\\Desktop\\jdk-8u191-linux-x64.tar.gz");

InputStream in = new BufferedInputStream(new FileInputStream(File));

FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/jdk-8u191-linux-x64.tar.gz"), new Progressable() {

@Override

public void progress() {

System.out.print("-");

}

});

IOUtils.copyBytes(in,out,4096);

}

//拷贝文件到本地目录

@Test

public void copyToLocalFile() throws Exception{

Path src = new Path("/hdfsapi/test/a.txt");

Path dst = new Path("e:/");

fileSystem.copyToLocalFile(src,dst);

}

//查看指定文件夹下所有文件

@Test

public void listFiles() throws Exception{

FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi/test"));

for (FileStatus file:statuses) {

System.out.println(file.isDirectory() ? "文件夹":"文件" + ":" + file.getPath());;

}

//递归查看指定文件夹下所有文件

@Test

public void listFilesRecursive() throws Exception{

RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/hdfsapi/"),true);

while (files.hasNext()){

LocatedFileStatus file = files.next();

System.out.println(file.getPath());

}

//查看文件块信息

@Test

public void getFileBlockLocations() throws Exception{

FileStatus status = fileSystem.getFileStatus(new Path("/hdfsapi/test/jdk-8u191-linux-x64.tar.gz"));

BlockLocation[] locations = fileSystem.getFileBlockLocations(status,0,status.getLen());

for (BlockLocation location: locations) {

for (String name:location.getNames()) {

System.out.println(name + ":" + location.getOffset() + ":" + location.getLength());

}

//删除文件

@Test

public void delete() throws Exception{

boolean res = fileSystem.delete(new Path("/hdfsapi/test/a.txt"));

System.out.println(res);

}

@After

public void tearDown(){

System.out.println("------------tearDown----------");

configuration = null;

fileSystem = null;

}