public class HDFSTool extends Object
| Modifier and Type | Field and Description |
|---|---|
static String |
DFS_BLOCKSIZE |
static String |
DFS_PERMISSIONS_ENABLED |
static String |
DFS_REPLICATION |
static String |
FS_DEFAULTFS |
static String |
IO_FILE_BUFFER_SIZE |
static String |
IO_SERIALIZATIONS |
static boolean |
USE_BINARYBLOCK_SERIALIZATION |
| Constructor and Description |
|---|
HDFSTool() |
| Modifier and Type | Method and Description |
|---|---|
static void |
addBinaryBlockSerializationFramework(org.apache.hadoop.conf.Configuration job) |
static void |
copyFileOnHDFS(String originalDir,
String newDir) |
static void |
createDirIfNotExistOnHDFS(org.apache.hadoop.fs.Path path,
String permissions) |
static void |
createDirIfNotExistOnHDFS(String dir,
String permissions) |
static void |
deleteFileIfExistOnHDFS(org.apache.hadoop.fs.Path outpath,
org.apache.hadoop.mapred.JobConf job) |
static void |
deleteFileIfExistOnHDFS(String dir) |
static void |
deleteFileIfExistOnLFS(org.apache.hadoop.fs.Path outpath,
org.apache.hadoop.mapred.JobConf job) |
static void |
deleteFileWithMTDIfExistOnHDFS(String fname) |
static long |
estimateNnzBasedOnFileSize(org.apache.hadoop.fs.Path path,
long rlen,
long clen,
int blen,
double factor) |
static boolean |
existsFileOnHDFS(String fname) |
static org.apache.hadoop.fs.FileStatus[] |
getDirectoryListing(String fname) |
static long |
getFilesizeOnHDFS(org.apache.hadoop.fs.Path path)
Returns the size of a file or directory on hdfs in bytes.
|
static org.apache.hadoop.fs.FSDataOutputStream |
getHDFSDataOutputStream(String filename,
boolean overwrite) |
static boolean |
isDirectory(String fname) |
static boolean |
isFileEmpty(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path dir) |
static boolean |
isHDFSFileEmpty(String dir) |
static void |
mergeIntoSingleFile(String originalDir,
String newFile) |
static String |
metaDataToString(Types.ValueType vt,
Types.ValueType[] schema,
Types.DataType dt,
DataCharacteristics dc,
Types.FileFormat fmt,
FileFormatProperties formatProperties,
PrivacyConstraint privacyConstraint) |
static boolean |
readBooleanFromHDFSFile(String filename) |
static double[] |
readColumnVectorFromHDFS(String dir,
Types.FileFormat fmt,
long rlen,
long clen,
int blen) |
static double |
readDoubleFromHDFSFile(String filename) |
static long |
readIntegerFromHDFSFile(String filename) |
static double[][] |
readMatrixFromHDFS(String dir,
Types.FileFormat fmt,
long rlen,
long clen,
int blen) |
static Object |
readObjectFromHDFSFile(String filename,
Types.ValueType vt) |
static ScalarObject |
readScalarObjectFromHDFSFile(String fname,
Types.ValueType vt) |
static String |
readStringFromHDFSFile(String filename) |
static void |
renameFileOnHDFS(String originalDir,
String newDir) |
static void |
writeBooleanToHDFS(boolean b,
String filename) |
static void |
writeDoubleToHDFS(double d,
String filename) |
static void |
writeIntToHDFS(long i,
String filename) |
static void |
writeMetaDataFile(String mtdfile,
Types.ValueType vt,
DataCharacteristics mc,
Types.FileFormat fmt) |
static void |
writeMetaDataFile(String mtdfile,
Types.ValueType vt,
DataCharacteristics dc,
Types.FileFormat fmt,
FileFormatProperties formatProperties) |
static void |
writeMetaDataFile(String mtdfile,
Types.ValueType vt,
DataCharacteristics dc,
Types.FileFormat fmt,
FileFormatProperties formatProperties,
PrivacyConstraint privacyConstraint) |
static void |
writeMetaDataFile(String mtdfile,
Types.ValueType vt,
DataCharacteristics mc,
Types.FileFormat fmt,
PrivacyConstraint privacyConstraint) |
static void |
writeMetaDataFile(String mtdfile,
Types.ValueType vt,
Types.ValueType[] schema,
Types.DataType dt,
DataCharacteristics mc,
Types.FileFormat fmt) |
static void |
writeMetaDataFile(String mtdfile,
Types.ValueType vt,
Types.ValueType[] schema,
Types.DataType dt,
DataCharacteristics dc,
Types.FileFormat fmt,
FileFormatProperties formatProperties) |
static void |
writeMetaDataFile(String mtdfile,
Types.ValueType vt,
Types.ValueType[] schema,
Types.DataType dt,
DataCharacteristics dc,
Types.FileFormat fmt,
FileFormatProperties formatProperties,
PrivacyConstraint privacyConstraint) |
static void |
writeMetaDataFile(String mtdfile,
Types.ValueType vt,
Types.ValueType[] schema,
Types.DataType dt,
DataCharacteristics mc,
Types.FileFormat fmt,
PrivacyConstraint privacyConstraint) |
static void |
writeObjectToHDFS(Object obj,
String filename) |
static void |
writeScalarMetaDataFile(String mtdfile,
Types.ValueType vt) |
static void |
writeScalarMetaDataFile(String mtdfile,
Types.ValueType vt,
PrivacyConstraint privacyConstraint) |
static void |
writeStringToHDFS(String s,
String filename) |
public static final String DFS_REPLICATION
public static final String IO_FILE_BUFFER_SIZE
public static final String IO_SERIALIZATIONS
public static final String DFS_BLOCKSIZE
public static final String DFS_PERMISSIONS_ENABLED
public static final String FS_DEFAULTFS
public static final boolean USE_BINARYBLOCK_SERIALIZATION
public static void addBinaryBlockSerializationFramework(org.apache.hadoop.conf.Configuration job)
public static boolean existsFileOnHDFS(String fname)
public static boolean isDirectory(String fname)
public static org.apache.hadoop.fs.FileStatus[] getDirectoryListing(String fname)
public static void deleteFileWithMTDIfExistOnHDFS(String fname) throws IOException
IOExceptionpublic static void deleteFileIfExistOnHDFS(String dir) throws IOException
IOExceptionpublic static void deleteFileIfExistOnHDFS(org.apache.hadoop.fs.Path outpath,
org.apache.hadoop.mapred.JobConf job)
throws IOException
IOExceptionpublic static void deleteFileIfExistOnLFS(org.apache.hadoop.fs.Path outpath,
org.apache.hadoop.mapred.JobConf job)
throws IOException
IOExceptionpublic static boolean isHDFSFileEmpty(String dir) throws IOException
IOExceptionpublic static boolean isFileEmpty(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path dir)
throws IOException
IOExceptionpublic static void renameFileOnHDFS(String originalDir, String newDir) throws IOException
IOExceptionpublic static void mergeIntoSingleFile(String originalDir, String newFile) throws IOException
IOExceptionpublic static void copyFileOnHDFS(String originalDir, String newDir) throws IOException
IOExceptionpublic static long estimateNnzBasedOnFileSize(org.apache.hadoop.fs.Path path,
long rlen,
long clen,
int blen,
double factor)
throws IOException
IOExceptionpublic static long getFilesizeOnHDFS(org.apache.hadoop.fs.Path path)
throws IOException
path - file system pathIOException - if IOException occurspublic static double readDoubleFromHDFSFile(String filename) throws IOException
IOExceptionpublic static long readIntegerFromHDFSFile(String filename) throws IOException
IOExceptionpublic static boolean readBooleanFromHDFSFile(String filename) throws IOException
IOExceptionpublic static String readStringFromHDFSFile(String filename) throws IOException
IOExceptionpublic static Object readObjectFromHDFSFile(String filename, Types.ValueType vt) throws IOException
IOExceptionpublic static ScalarObject readScalarObjectFromHDFSFile(String fname, Types.ValueType vt)
public static void writeDoubleToHDFS(double d,
String filename)
throws IOException
IOExceptionpublic static void writeIntToHDFS(long i,
String filename)
throws IOException
IOExceptionpublic static void writeBooleanToHDFS(boolean b,
String filename)
throws IOException
IOExceptionpublic static void writeStringToHDFS(String s, String filename) throws IOException
IOExceptionpublic static void writeObjectToHDFS(Object obj, String filename) throws IOException
IOExceptionpublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics mc, Types.FileFormat fmt) throws IOException
IOExceptionpublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics mc, Types.FileFormat fmt, PrivacyConstraint privacyConstraint) throws IOException
IOExceptionpublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics mc, Types.FileFormat fmt) throws IOException
IOExceptionpublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics mc, Types.FileFormat fmt, PrivacyConstraint privacyConstraint) throws IOException
IOExceptionpublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties) throws IOException
IOExceptionpublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties, PrivacyConstraint privacyConstraint) throws IOException
IOExceptionpublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties) throws IOException
IOExceptionpublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties, PrivacyConstraint privacyConstraint) throws IOException
IOExceptionpublic static void writeScalarMetaDataFile(String mtdfile, Types.ValueType vt) throws IOException
IOExceptionpublic static void writeScalarMetaDataFile(String mtdfile, Types.ValueType vt, PrivacyConstraint privacyConstraint) throws IOException
IOExceptionpublic static String metaDataToString(Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties, PrivacyConstraint privacyConstraint) throws org.apache.wink.json4j.JSONException, DMLRuntimeException
org.apache.wink.json4j.JSONExceptionDMLRuntimeExceptionpublic static double[][] readMatrixFromHDFS(String dir, Types.FileFormat fmt, long rlen, long clen, int blen) throws IOException, DMLRuntimeException
IOExceptionDMLRuntimeExceptionpublic static double[] readColumnVectorFromHDFS(String dir, Types.FileFormat fmt, long rlen, long clen, int blen) throws IOException, DMLRuntimeException
IOExceptionDMLRuntimeExceptionpublic static void createDirIfNotExistOnHDFS(String dir, String permissions) throws IOException
IOExceptionpublic static void createDirIfNotExistOnHDFS(org.apache.hadoop.fs.Path path,
String permissions)
throws IOException
IOExceptionpublic static org.apache.hadoop.fs.FSDataOutputStream getHDFSDataOutputStream(String filename, boolean overwrite) throws IOException
IOExceptionCopyright © 2021 The Apache Software Foundation. All rights reserved.