實驗3-熟悉常用的HDFS操作-答案

1、精選優(yōu)質(zhì)文檔-傾情為你奉上實驗2熟悉常用的HDFS操作1 實驗?zāi)康?.理解HDFS在Hadoop體系結(jié)構(gòu)中的角色；2.熟練使用HDFS操作常用的Shell命令；3.熟悉HDFS操作常用的Java API。2 實驗平臺操作系統(tǒng)：LinuxHadoop版本：或以上版本JDK版本：或以上版本Java IDE：Eclipse3 實驗內(nèi)容和要求1. 編程實現(xiàn)以下指定功能，并利用Hadoop提供的Shell命令完成相同任務(wù)：提示：1) 部分Shell命令的參數(shù)路徑只能是本地路徑或者HDFS路徑。2) 若Shell命令的參數(shù)既可以是本地路徑，也可以是HDFS路徑時，務(wù)必注意區(qū)分。為保證操作正確，可指定路徑前

2、綴 或者 注意區(qū)分相對路徑與絕對路徑3) 具體命令的說明可參考教材或 （1） 向HDFS中上傳任意文本文件，如果指定的文件在HDFS中已經(jīng)存在，由用戶指定是追加到原有文件末尾還是覆蓋原有的文件；Shell命令:檢查文件是否存在: ./hdfs dfs -test -e (執(zhí)行完這一句不會輸出結(jié)果，需要繼續(xù)輸入命令" echo $")追加命令: ./hdfs dfs -appendToFile 覆蓋命令1: ./hdfs dfs -copyFromLocal -f 覆蓋命令2: ./hdfs dfs -cp -f 也可以使用如下命令實現(xiàn)：（如下代碼可視為一行代碼，在終端中輸入

3、第一行代碼后，直到輸入 fi 才會真正執(zhí)行）：if $(./hdfs dfs -test -e ;then $(./hdfs dfs -appendToFile ;else $(./hdfs dfs -copyFromLocal -f ;fiJava代碼：import .*;public class HDFSApi /* * 判斷路徑是否存在 */ public static boolean test(Configuration conf, String path) throws IOException FileSystem fs = (conf); return (new Path(path

4、); /* * 復(fù)制文件到指定路徑 * 若路徑已存在，則進(jìn)行覆蓋 */ public static void copyFromLocalFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOException FileSystem fs = (conf); Path localPath = new Path(localFilePath); Path remotePath = new Path(remoteFilePath); /* 第一個參數(shù)表示是否刪除源文件，第二個參數(shù)表示是否覆蓋 */

5、(false, true, localPath, remotePath); (); /* * 追加文件內(nèi)容 */ public static void appendToFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOException FileSystem fs = (conf); Path remotePath = new Path(remoteFilePath); /* 創(chuàng)建一個文件讀入流 */ FileInputStream in = new FileInputStream(lo

6、calFilePath); /* 創(chuàng)建一個文件輸出流，輸出的內(nèi)容將追加到文件末尾 */ FSDataOutputStream out = (remotePath); /* 讀寫文件內(nèi)容 */ byte data = new byte1024; int read = -1; while ( (read = (data) > 0 ) (data, 0, read); (); (); (); /* * 主函數(shù) */public static void main(String args) Configuration conf = new Configuration(); (""

7、;,"");String localFilePath = "/home/hadoop/" ; else + " 不存在.");/* 進(jìn)行處理 */if ( !fileExists) （2）hdfs dfs -test -e ;then $(./hdfs dfs -copyToLocal ./; else $(./hdfs dfs -copyToLocal ./; fiJava代碼：import .*;public class HDFSApi /* * 下載文件到本地 * 判斷本地路徑是否已存在，若已存在，則自動進(jìn)行重命名 */ pub

8、lic static void copyToLocal(Configuration conf, String remoteFilePath, String localFilePath) throws IOException FileSystem fs = (conf); Path remotePath = new Path(remoteFilePath); File f = new File(localFilePath); /* 如果文件名存在，自動重命名(在文件名后面加上 _0, _1 .) */ if () + " 已存在."); Integer i = 0; whil

9、e (true) f = new File(localFilePath + "_" + (); if (!() localFilePath = localFilePath + "_" + (); break; "將重新命名為: " + localFilePath); （3）hdfs dfs -cat Java代碼：import .*;public class HDFSApi /* * 讀取文件內(nèi)容 */ public static void cat(Configuration conf, String remoteFilePath)

10、throws IOException FileSystem fs = (conf); Path remotePath = new Path(remoteFilePath); FSDataInputStream in = (remotePath); BufferedReader d = new BufferedReader(new InputStreamReader(in); String line = null; while ( (line = () != null ) (); (); (); /* * 主函數(shù) */public static void main(String args) Co

11、nfiguration conf = new Configuration(); ("","");String remoteFilePath = "/user/hadoop/" （4）hdfs dfs -ls -h Java代碼：import .*;import class HDFSApi /* * 顯示指定文件的信息 */ public static void ls(Configuration conf, String remoteFilePath) throws IOException FileSystem fs = (conf);

12、 Path remotePath = new Path(remoteFilePath); FileStatus fileStatuses = (remotePath); for (FileStatus s : fileStatuses) "路徑: " + ().toString(); "權(quán)限: " + ().toString(); "大小: " + (); /* 返回的是時間戳,轉(zhuǎn)化為時間日期格式 */ Long timeStamp = (); SimpleDateFormat format = new SimpleDateForma

13、t("yyyy-MM-dd HH:mm:ss"); String date = (timeStamp); "時間: " + date); (); /* * 主函數(shù) */public static void main(String args) Configuration conf = new Configuration(); ("","");String remoteFilePath = "/user/hadoop/" （5）hdfs dfs -ls -R -h /user/hadoopJava代

14、碼：import .*;import class HDFSApi /* * 顯示指定文件夾下所有文件的信息（遞歸） */ public static void lsDir(Configuration conf, String remoteDir) throws IOException FileSystem fs = (conf); Path dirPath = new Path(remoteDir); /* 遞歸獲取目錄下的所有文件 */ RemoteIterator<LocatedFileStatus> remoteIterator = (dirPath, true); /* 輸

15、出每個文件的信息 */ while () FileStatus s = (); "路徑: " + ().toString(); "權(quán)限: " + ().toString(); "大小: " + (); /* 返回的是時間戳,轉(zhuǎn)化為時間日期格式 */ Long timeStamp = (); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); String date = (timeStamp); "時間: "

16、 + date); (); /* * 主函數(shù) */public static void main(String args) Configuration conf = new Configuration(); ("","");String remoteDir = "/user/hadoop" （6）hdfs dfs -test -d dir1/dir2);then $(./hdfs dfs -touchz dir1/dir2/filename); else $(./hdfs dfs -mkdir -p dir1/dir2 &&a

17、mp; hdfs dfs -touchz dir1/dir2/filename); fi刪除文件：./hdfs dfs -rm dir1/dir2/filenameJava代碼：import .*;public class HDFSApi /* * 判斷路徑是否存在 */ public static boolean test(Configuration conf, String path) throws IOException FileSystem fs = (conf); return (new Path(path); /* * 創(chuàng)建目錄 */ public static boolean m

18、kdir(Configuration conf, String remoteDir) throws IOException FileSystem fs = (conf); Path dirPath = new Path(remoteDir); boolean result = (dirPath); (); return result; /* * 創(chuàng)建文件 */ public static void touchz(Configuration conf, String remoteFilePath) throws IOException FileSystem fs = (conf); Path r

19、emotePath = new Path(remoteFilePath); FSDataOutputStream outputStream = (remotePath); (); (); /* * 刪除文件 */ public static boolean rm(Configuration conf, String remoteFilePath) throws IOException FileSystem fs = (conf); Path remotePath = new Path(remoteFilePath); boolean result = (remotePath, false);

20、(); return result; /* * 主函數(shù) */public static void main(String args) Configuration conf = new Configuration(); ("","");String remoteFilePath = "/user/hadoop/input/" （7）hdfs dfs -mkdir -p dir1/dir2刪除目錄（如果目錄非空則會提示not empty，不執(zhí)行刪除）：./hdfs dfs -rmdir dir1/dir2強制刪除目錄：./hdfs dfs

21、 -rm -R dir1/dir2Java代碼：import .*;public class HDFSApi /* * 判斷路徑是否存在 */ public static boolean test(Configuration conf, String path) throws IOException FileSystem fs = (conf); return (new Path(path); /* * 判斷目錄是否為空 * true: 空，false: 非空 */ public static boolean isDirEmpty(Configuration conf, String remo

22、teDir) throws IOException FileSystem fs = (conf); Path dirPath = new Path(remoteDir); RemoteIterator<LocatedFileStatus> remoteIterator = (dirPath, true); return !(); /* * 創(chuàng)建目錄 */ public static boolean mkdir(Configuration conf, String remoteDir) throws IOException FileSystem fs = (conf); Path d

23、irPath = new Path(remoteDir); boolean result = (dirPath); (); return result; /* * 刪除目錄 */ public static boolean rmDir(Configuration conf, String remoteDir) throws IOException FileSystem fs = (conf); Path dirPath = new Path(remoteDir); /* 第二個參數(shù)表示是否遞歸刪除所有文件 */ boolean result = (dirPath, true); (); ret

24、urn result; /* * 主函數(shù) */public static void main(String args) Configuration conf = new Configuration(); ("","");String remoteDir = "/user/hadoop/input" （8）hdfs dfs -appendToFile 追加到文件開頭：（由于沒有直接的命令可以操作，方法之一是先移動到本地進(jìn)行操作，再進(jìn)行上傳覆蓋）：./hdfs dfs -get cat >> ./hdfs dfs -copyF

25、romLocal -f Java代碼：import .*;public class HDFSApi /* * 判斷路徑是否存在 */ public static boolean test(Configuration conf, String path) throws IOException FileSystem fs = (conf); return (new Path(path); /* * 追加文本內(nèi)容 */ public static void appendContentToFile(Configuration conf, String content, String remoteFil

26、ePath) throws IOException FileSystem fs = (conf); Path remotePath = new Path(remoteFilePath); /* 創(chuàng)建一個文件輸出流，輸出的內(nèi)容將追加到文件末尾 */ FSDataOutputStream out = (remotePath); (); (); (); /* * 追加文件內(nèi)容 */ public static void appendToFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOExce

27、ption FileSystem fs = (conf); Path remotePath = new Path(remoteFilePath); /* 創(chuàng)建一個文件讀入流 */ FileInputStream in = new FileInputStream(localFilePath); /* 創(chuàng)建一個文件輸出流，輸出的內(nèi)容將追加到文件末尾 */ FSDataOutputStream out = (remotePath); /* 讀寫文件內(nèi)容 */ byte data = new byte1024; int read = -1; while ( (read = (data) > 0

28、) (data, 0, read); (); (); (); /* * 移動文件到本地 * 移動后，刪除源文件 */ public static void moveToLocalFile(Configuration conf, String remoteFilePath, String localFilePath) throws IOException FileSystem fs = (conf); Path remotePath = new Path(remoteFilePath); Path localPath = new Path(localFilePath); (remotePath,

29、 localPath); /* * 創(chuàng)建文件 */ public static void touchz(Configuration conf, String remoteFilePath) throws IOException FileSystem fs = (conf); Path remotePath = new Path(remoteFilePath); FSDataOutputStream outputStream = (remotePath); (); (); /* * 主函數(shù) */public static void main(String args) Configuration

30、conf = new Configuration(); ("","");String remoteFilePath = "/user/hadoop/" （9）hdfs dfs -rm Java命令：import .*;public class HDFSApi /* * 刪除文件 */ public static boolean rm(Configuration conf, String remoteFilePath) throws IOException FileSystem fs = (conf); Path remotePath

31、= new Path(remoteFilePath); boolean result = (remotePath, false); (); return result; /* * 主函數(shù) */public static void main(String args) Configuration conf = new Configuration(); ("","");String remoteFilePath = "/user/hadoop/" （10）hdfs dfs -rmdir dir1/dir2強制刪除目錄：./hdfs dfs

32、-rm -R dir1/dir2Java代碼：import .*;public class HDFSApi /* * 判斷目錄是否為空 * true: 空，false: 非空 */ public static boolean isDirEmpty(Configuration conf, String remoteDir) throws IOException FileSystem fs = (conf); Path dirPath = new Path(remoteDir); RemoteIterator<LocatedFileStatus> remoteIterator = (d

33、irPath, true); return !(); /* * 刪除目錄 */ public static boolean rmDir(Configuration conf, String remoteDir, boolean recursive) throws IOException FileSystem fs = (conf); Path dirPath = new Path(remoteDir); /* 第二個參數(shù)表示是否遞歸刪除所有文件 */ boolean result = (dirPath, recursive); (); return result; /* * 主函數(shù) */pub

34、lic static void main(String args) Configuration conf = new Configuration(); ("","");String remoteDir = "/user/hadoop/input" （11）hdfs dfs -mv Java代碼：import .*;public class HDFSApi /* * 移動文件 */ public static boolean mv(Configuration conf, String remoteFilePath, String rem

35、oteToFilePath) throws IOException FileSystem fs = (conf); Path srcPath = new Path(remoteFilePath); Path dstPath = new Path(remoteToFilePath); boolean result = (srcPath, dstPath); (); return result; /* * 主函數(shù) */public static void main(String args) Configuration conf = new Configuration(); ("","");String remoteFilePath = "" 2.;public class MyFSDa