hadoop 平衡节点数据
在集群运行中,会发现在不同的节点中空间利用率是不同的,甚至会造成有的节点数据快存满了,有的利用率还不… 继续阅读 hadoop 平衡节点数据
为您提供常用小插件
在集群运行中,会发现在不同的节点中空间利用率是不同的,甚至会造成有的节点数据快存满了,有的利用率还不… 继续阅读 hadoop 平衡节点数据
hdp-configuration-utils.py Hadoop的内存配置有两种方法:利用手动安装… 继续阅读 Hadoop的内存配置
1 2 3 4 5 6 7 |
<dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>3.1.3</version> </dependency> |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 |
package com.imooc.bigdata.hdfs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Progressable; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.*; import java.net.URI; import java.net.URISyntaxException; public class HDFSApp { public static final String HDFS_PATH = "hdfs://192.168.64.131:9000"; FileSystem fileSystem = null; Configuration configuration = null; @Before public void setUp() throws URISyntaxException, IOException, InterruptedException { System.out.println("------------setup----------"); configuration = new Configuration(); //设置副本个数 configuration.set("dfs.replication","1"); fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration,"root"); } //创建目录 @Test public void mkdir() throws IOException { System.out.println("------------mkdir----------"); fileSystem.mkdirs(new Path("/hdfsapi/test")); } //创建并写入文件 @Test public void create() throws Exception{ FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/b.txt")); // FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/a.txt")); out.writeUTF("hello pk dfs.replication 1"); out.flush(); out.close(); } //读取文件 @Test public void text() throws IOException { FSDataInputStream FSDataInputStream = fileSystem.open(new Path("/hdfsapi/test/a.txt")); IOUtils.copyBytes(FSDataInputStream,System.out,1024); } //更改文件名 @Test public void rename() throws Exception{ Path oldPath = new Path("/hdfsapi/test/b.txt"); Path newPath = new Path("/hdfsapi/test/c.txt"); boolean res = fileSystem.rename(oldPath,newPath); System.out.println(res); } //拷贝本地文件到hdfs系统 @Test public void copyFromLocalFile() throws Exception{ Path src = new Path("E:\\download\\count.exe"); Path dst = new Path("/hdfsapi/test02/"); fileSystem.copyFromLocalFile(src,dst); } //拷贝大文件 @Test public void copyFromLocalBigFile() throws Exception{ File File = new File("C:\\Users\\Administrator\\Desktop\\jdk-8u191-linux-x64.tar.gz"); InputStream in = new BufferedInputStream(new FileInputStream(File)); FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/jdk-8u191-linux-x64.tar.gz"), new Progressable() { @Override public void progress() { System.out.print("-"); } }); IOUtils.copyBytes(in,out,4096); } //拷贝文件到本地目录 @Test public void copyToLocalFile() throws Exception{ Path src = new Path("/hdfsapi/test/a.txt"); Path dst = new Path("e:/"); fileSystem.copyToLocalFile(src,dst); } //查看指定文件夹下所有文件 @Test public void listFiles() throws Exception{ FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi/test")); for (FileStatus file:statuses) { System.out.println(file.isDirectory() ? "文件夹":"文件" + ":" + file.getPath());; } } //递归查看指定文件夹下所有文件 @Test public void listFilesRecursive() throws Exception{ RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/hdfsapi/"),true); while (files.hasNext()){ LocatedFileStatus file = files.next(); System.out.println(file.getPath()); } } //查看文件块信息 @Test public void getFileBlockLocations() throws Exception{ FileStatus status = fileSystem.getFileStatus(new Path("/hdfsapi/test/jdk-8u191-linux-x64.tar.gz")); BlockLocation[] locations = fileSystem.getFileBlockLocations(status,0,status.getLen()); for (BlockLocation location: locations) { for (String name:location.getNames()) { System.out.println(name + ":" + location.getOffset() + ":" + location.getLength()); } } } //删除文件 @Test public void delete() throws Exception{ boolean res = fileSystem.delete(new Path("/hdfsapi/test/a.txt")); System.out.println(res); } @After public void tearDown(){ System.out.println("------------tearDown----------"); configuration = null; fileSystem = null; } } |