試驗(yàn)熟悉常用的HDFS操作-答案

1、實(shí)驗(yàn)2熟悉常用的HDFS操作1實(shí)驗(yàn)?zāi)康?. 理解HDFS在Hadoop體系結(jié)構(gòu)中的角色；2. 熟練使用HDFS操作常用的Shell命令；3. 熟悉HDFS操作常用的Java API。2實(shí)驗(yàn)平臺操作系統(tǒng)：Lin uxHadoop版本：或以上版本JDK版本：或以上版本Java IDE : Eclipse3實(shí)驗(yàn)內(nèi)容和要求1.編程實(shí)現(xiàn)以下指定功能，并利用 Hadoop提供的Shell命令完成相同任務(wù)： 提示：1）部分Shell命令的參數(shù)路徑只能是本地路徑或者HDFS路徑。2）若Shell命令的參數(shù)既可以是本地路徑，也可以是HDFS路徑時，務(wù)必注意區(qū)分。為保證操作正確，可指定路徑前綴或者注意區(qū)分相對路徑

2、與絕對路徑3）具體命令的說明可參考教材或（1） 向HDFS中上傳任意文本文件，如果指定的文件在 HDFS已經(jīng)存在，由用戶指定是追加到原有文件末尾還是覆蓋原有的文件；Shell命令：檢查文件是否存在：./hdfs dfs -test -e （執(zhí)行完這一句不會輸出結(jié)果，需要繼續(xù)輸入命令echo $）追加命令:./hdfs dfs -appendToFile覆蓋命令 1: ./hdfs dfs -copyFromLocal -f覆蓋命令 2: ./hdfs dfs -cp -f也可以使用如下命令實(shí)現(xiàn)：（如下代碼可視為一行代碼，在終端中輸入第一行代碼后，直到輸入fi才會真正執(zhí)行）：if $（./hdf

3、s dfs -test -e ;then $（./hdfs dfs -appe ndToFile ;else $（./hdfs dfs -copyFromLocal -f ;fiJava代碼：import .*;public class HDFSApi /*判斷路徑是否存在*/public static boolean test(Configurationconf. String path) throws lOExceptionFileSystem fs = (conf);return (new Path(path);/* 復(fù)制文件到指定路徑* 若路徑已存在，則進(jìn)行覆蓋*/conf. Stri

4、ngpublic static void copyFromLocalFile(ConfigurationStri ng remoteFilePath) throws IOExcepti on FileSystem fs = (conf);Path localPath = new Path(localFilePath);Path remotePath = new Path(remoteFilePath);/*第一個參數(shù)表示是否刪除源文件，第二個參數(shù)表示是否覆蓋(false, true, localPath, remotePath);();localFilePath,*/* 追加文件內(nèi)容*/pub

5、lic static void appe ndToFile(C on figurati onStri ng remoteFilePath) throws IOExcepti on FileSystem fs = (conf);Path remotePath = new Path(remoteFilePath);/*創(chuàng)建一個文件讀入流*/FilelnputStream in = new FilelnputStream(localFilePath);/*創(chuàng)建一個文件輸出流，輸出的內(nèi)容將追加到文件末尾FSDataOutputStream out = (remotePath);/*讀寫文件內(nèi)容*/by

6、te data = new byte1024;int read = -1;while ( (read = (data) 0 ) (data, 0, read);conf. StringlocalFilePath,*/()；()；()；*主函數(shù)*/public static void main( Stri ng args) Con figurati on conf = new Con figurati on(); (”，”)；Stri ng localFilePath = /home/hadoop/; else + 不存在.);/Java代碼：import .*;public class HDF

7、SApi /* 下載文件到本地 判斷本地路徑是否已存在，若已存在，則自動進(jìn)行重命名進(jìn)行處理*/if ( !fileExis ts) (1)hdfs dfs -test -e ;then $(./hdfs dfs -copyToLocal ./;else $(./hdfs dfs -copyToLocal ./;fihdfs dfs -catJava代碼： public class HDFSApi /*讀取文件內(nèi)容*/conf, Stri ng remoteFilePath) throwspublic static void cat(C on figurati onIOExcepti on Fi

8、leSystem fs = (conf);Path remotePath = new Path(remoteFilePath);FSDataI nputStream in = (remotePath);BufferedReader d = new BufferedReader( new In putStreamReader(i n); Stri ng line = n ull;while ( (li ne = () != null ) ();();();/*主函數(shù)*/public static void main( Stri ng args) Con figurati on conf = ne

9、w Con figurati on();(”，”)；Stri ng remoteFilePath = /user/hadoop/;(3)hdfs dfs -ls -h*/publicstatic voidls(C on figurati onconf, String remoteFilePath) throwslOExcepti on FileSystem fs = (conf);Path remotePath = new Path(remoteFilePath);FileStatus fileStatuses = (remotePath);for (FileStatus s : fileSt

10、atuses) 路徑:+ ().toString();權(quán)限:+ O.toString();大小:+ ();/顯示指定文件夾下所有文件的信息(遞歸)返回的是時間戳，轉(zhuǎn)化為時間日期格式 */Long timeStamp =();SimpleDateFormat format = new SimpleDateFormat(yyyy-MM-dd HH:mm:ss);Stri ng date = (timeStamp);時間:+ date);();/Java代碼： public class HDFSApi /*/public static boolean test(Configurationconf,

11、String path) throws IOExceptionFileSystem fs = (conf);return (new Path(path);/*主函數(shù)*/public static void main(String args) Con figurati on conf = new Con figurati on();(,);Stri ng remoteFilePath = /user/hadoop/;(4)hdfs dfs -ls -R -h /user/hadoopII大小:+ ()；/創(chuàng)建目錄返回的是時間戳，轉(zhuǎn)化為時間日期格式*/Long timeStamp =();Simp

12、leDateFormat format = new SimpleDateFormat(yyyy-MM-dd HH:mm:ss);Stri ng date = (timeStamp);時間:+ date);()；/Java代碼：import .*;*主函數(shù)*/public static void main(String args) Con figurati on conf = new Con figurati on();(,);Stri ng remoteDir = /user/hadoop;(5) hdfs dfs -test -d dir1/dir2);then $(./hdfs dfs -

13、touchz dir1/dir2/file name);else $(./hdfs dfs -mkdir -p dir1/dir2 & hdfs dfs -touchz dir1/dir2/file name); fi判斷路徑是否存在conf,Stri ngremoteDir) throws刪除文件：./hdfs dfs -rm dir1/dir2/file nameboolea n result = (dirPath);()；return result;/*創(chuàng)建文件*/public static void touchz(C on figurati on conf, Stri ng remot

14、eFilePath) throws lOExcepti on FileSystem fs = (conf);Path remotePath = new Path(remoteFilePath);FSDataOutputStream outputStream = (remotePath);();();/*刪除文件*/public static boolea n rm(C on figurati on conf, Stri ng remoteFilePath) throws lOExcepti on FileSystem fs = (conf);Path remotePath = new Path

15、(remoteFilePath);boolea n result = (remotePath, false);();return result;/*主函數(shù)*/public static void main(String args) Con figurati on conf = new Con figurati on();(,);Stri ng remoteFilePath = /user/hadoop/i nput/;(6)hdfs dfs -mkdir -p dir1/dir2刪除目錄(如果目錄非空則會提示not empty,不執(zhí)行刪除)：./hdfs dfs -rmdir dir1/dir

16、2強(qiáng)制刪除目錄：./hdfs dfs -rm -R dir1/dir2public class HDFSApi /*判斷路徑是否存在*/public staticboolean test(Configurationconf. String path)FileSystem fs = (conf);return (new Path(path);throws lOException* 判斷目錄是否為空* true:空，false:非空*/public static boolean isDirEmpty(Configurationconf. StringlOExcepti on FileSystem

17、fs = (conf);Path dirPath = new Path(remoteDir);Remotelterator remoteiterator = (dirPath, true); return !();remoteDir) throws/*創(chuàng)建目錄*/public static boolea n mkdir(C on figurati on lOExcepti on FileSystem fs = (conf);Path dirPath = new Path(remoteDir);boolea n result = (dirPath);();return result;conf.

18、StringremoteDir) throws/*刪除目錄*/public staticboolea nrmDir(C on figurati onconf. StringlOExcepti on FileSystem fs = (conf);Path dirPath = new Path(remoteDir);/*第二個參數(shù)表示是否遞歸刪除所有文件*/boolea n result = (dirPath, true);();remoteDir) throwspublic staticboolean test(ConfigurationFileSystem fs = (conf);return

19、 (new Path(path);conf. String path) throws IOExceptionretur n result;/*/*主函數(shù)*/public static void main( Stri ng args) Con figurati on conf = new Con figurati on();(”，”)；String remoteDir = /user/hadoop/input;(7) hdfs dfs -appe ndToFile追加到文件開頭：（由于沒有直接的命令可以操作，方法之一是先移動到本地進(jìn)行操作，再進(jìn)行上傳覆蓋）./hdfs dfs -getcat .

20、/hdfs dfs -copyFromLocal -f* 追加文本內(nèi)容*/public static void appe ndConten tToFile(C on figurati on conf. String content, Stri ng remoteFilePath) throws IOExcepti on FileSystem fs = (conf);Path remotePath = new Path(remoteFilePath);/判斷路徑是否存在創(chuàng)建一個文件輸出流，輸出的內(nèi)容將追加到文件末尾*/FSDataOutputStream out = (remotePath);(

21、);();();*追加文件內(nèi)容*/public static void appe ndToFile(C on figurati on Stri ng remoteFilePath) throws IOExcepti on FileSystem fs = (conf);Path remotePath = new Path(remoteFilePath);/*創(chuàng)建一個文件讀入流 */conf. StringlocalFilePath,FilelnputStream in = new FilelnputStream(localFilePath);/*創(chuàng)建一個文件輸出流，輸出的內(nèi)容將追加到文件末尾*/

22、FSDataOutputStream out = (remotePath);/*讀寫文件內(nèi)容*/byte data = new byte1024;int read = -1;while ( (read = (data) 0 ) (data, 0, read);();();();* 移動文件到本地* 移動后，刪除源文件*/public static void moveToLocalFile(Configuration Stri ng localFilePath) throws lOException FileSystem fs = (conf);Path remotePath = new Pat

23、h(remoteFilePath);Path localPath = new Path(localFilePath); (remotePath, localPath);conf. StringremoteFilePath,/*創(chuàng)建文件*/public static void touchz(C on figurati on conf, Stri ng remoteFilePath) throws IOExcepti on FileSystem fs = (conf);Path remotePath = new Path(remoteFilePath);FSDataOutputStream out

24、putStream = (remotePath);();();/*主函數(shù)*/public static void main(String args) Con figurati on conf = new Con figurati on();(”，”)；Stri ng remoteFilePath = /user/hadoop/;(8) hdfs dfs -rmJava命令： public class HDFSApi /*刪除文件*/public static boolea n rm(C on figurati on conf, Stri ng remoteFilePath) throws IO

25、Excepti on FileSystem fs = (conf);Path remotePath = new Path(remoteFilePath);boolea n result = (remotePath, false);();return result;/*主函數(shù)*/public static void main(String args) Con figurati on conf = new Con figurati on();(,);Stri ng remoteFilePath = /user/hadoop/;(9) hdfs dfs -rmdir dir1/dir2強(qiáng)制刪除目錄：

26、./hdfs dfs -rm -R dir1/dir2Java代碼： import .*;public class HDFSApi /*/public static boolea n mv(Con figurati on conf, Stri ng remoteFilePath, String remoteToFilePath) throws lOExcepti on * 判斷目錄是否為空* true:空，false:非空*/conf, Stri ng remoteDir) throwspublic static boolean isDirEmpty(Configuration lOExcep

27、ti on FileSystem fs = (conf);Path dirPath = new Path(remoteDir);Remotelterator remoteIterator = (dirPath, true); return !();/*刪除目錄*/public staticboolea n rmDir(C on figuratio nconf, Stri ng remoteDir, boolea nrecursive) throws lOExcepti on FileSystem fs = (conf);*/Path dirPath = new Path(remoteDir);

28、/Java代碼： import .* public class HDFSApi /* 移動文件第二個參數(shù)表示是否遞歸刪除所有文件boolea n result = (dirPath, recursive);();return result;/*主函數(shù)*/public static void main(String args) Con figurati on conf = new Con figurati on();(,);String remoteDir = /user/hadoop/input;(10) hdfs dfs -mvFileSystem fs = (conf);Path srcP

29、ath = new Path(remoteFilePath);Path dstPath = new Path(remoteToFilePath); boolea n result = (srcPath, dstPath);()；return result;*主函數(shù)*/public static void main( Stri ng args) Con figurati on conf = new Con figurati on(); (,);Stri ng remoteFilePath =;2.public class MyFSData In putStream exte nds FSData

30、 In putStream public MyFSDatalnputStream(InputStream in) super(i n);實(shí)現(xiàn)按行讀取每次讀入一個字符，遇到n結(jié)束，返回一行內(nèi)容*/public static String readli ne(BufferedReader br) throws lOExcepti on char data = new char1024;int read = -1;int off = 0; quals(n) ) off += 1;break;off += 1;if (off 0) return (data); else return n ull;/*

31、讀取文件內(nèi)容*/public static void cat(C on figuratio n conf. String remoteFilePath) throws 10 Excepti on FileSystem fs = (conf);Path remotePath = new Path(remoteFilePath);FSDatal nputStream in = (remotePath);BufferedReader br = new BufferedReader( new In putStreamReader(i n);Stri ng line = n ull;while ( (l

32、i ne = (br) != null ) ();();(); /*主函數(shù)*/public static void main(String args) Con figurati on conf = new Con figurati on();(,);Stri ng remoteFilePath = /user/hadoop/;3.import class HDFSApi static(new FsUrlStreamHa ndlerFactory();/*主函數(shù)*/public static void main(String args) throws Exception Stri ng remoteFilePath = ;pen Strea