public class SparkUtils extends Object
| Modifier and Type | Field and Description | 
|---|---|
| static org.apache.spark.storage.StorageLevel | DEFAULT_TMP | 
| static ExecutorService | triggerRDDPool | 
| Constructor and Description | 
|---|
| SparkUtils() | 
| Modifier and Type | Method and Description | 
|---|---|
| static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> | cacheBinaryCellRDD(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> input) | 
| static void | checkSparsity(String varname,
             ExecutionContext ec) | 
| static DataCharacteristics | computeDataCharacteristics(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> input)Utility to compute dimensions and non-zeros in a given RDD of binary cells. | 
| static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> | copyBinaryBlockMatrix(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> in)Creates a partitioning-preserving deep copy of the input matrix RDD, where 
 the indexes and values are copied. | 
| static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> | copyBinaryBlockMatrix(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> in,
                     boolean deep)Creates a partitioning-preserving copy of the input matrix RDD. | 
| static org.apache.spark.api.java.JavaPairRDD<TensorIndexes,BasicTensorBlock> | copyBinaryBlockTensor(org.apache.spark.api.java.JavaPairRDD<TensorIndexes,BasicTensorBlock> in)Creates a partitioning-preserving deep copy of the input tensor RDD, where
 the indexes and values are copied. | 
| static org.apache.spark.api.java.JavaPairRDD<TensorIndexes,BasicTensorBlock> | copyBinaryBlockTensor(org.apache.spark.api.java.JavaPairRDD<TensorIndexes,BasicTensorBlock> in,
                     boolean deep)Creates a partitioning-preserving copy of the input tensor RDD. | 
| static List<scala.Tuple2<Long,FrameBlock>> | fromIndexedFrameBlock(List<Pair<Long,FrameBlock>> in) | 
| static scala.Tuple2<Long,FrameBlock> | fromIndexedFrameBlock(Pair<Long,FrameBlock> in) | 
| static scala.Tuple2<MatrixIndexes,MatrixBlock> | fromIndexedMatrixBlock(IndexedMatrixValue in) | 
| static List<scala.Tuple2<MatrixIndexes,MatrixBlock>> | fromIndexedMatrixBlock(List<IndexedMatrixValue> in) | 
| static Pair<MatrixIndexes,MatrixBlock> | fromIndexedMatrixBlockToPair(IndexedMatrixValue in) | 
| static List<Pair<MatrixIndexes,MatrixBlock>> | fromIndexedMatrixBlockToPair(List<IndexedMatrixValue> in) | 
| static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> | getEmptyBlockRDD(org.apache.spark.api.java.JavaSparkContext sc,
                DataCharacteristics mc)Creates an RDD of empty blocks according to the given matrix characteristics. | 
| static long | getNonZeros(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> input) | 
| static long | getNonZeros(MatrixObject mo) | 
| static int | getNumPreferredPartitions(DataCharacteristics dc) | 
| static int | getNumPreferredPartitions(DataCharacteristics dc,
                         boolean outputEmptyBlocks) | 
| static int | getNumPreferredPartitions(DataCharacteristics dc,
                         org.apache.spark.api.java.JavaPairRDD<?,?> in) | 
| static String | getPrefixFromSparkDebugInfo(String line) | 
| static String | getStartLineFromSparkDebugInfo(String line) | 
| static boolean | isHashPartitioned(org.apache.spark.api.java.JavaPairRDD<?,?> in)Indicates if the input RDD is hash partitioned, i.e., it has a partitioner
 of type  org.apache.spark.HashPartitioner. | 
| static void | postprocessUltraSparseOutput(MatrixObject mo,
                            DataCharacteristics mcOut) | 
| static void | shutdownPool() | 
| static Pair<Long,FrameBlock> | toIndexedFrameBlock(scala.Tuple2<Long,FrameBlock> in) | 
| static List<Pair<Long,Long>> | toIndexedLong(List<scala.Tuple2<Long,Long>> in) | 
| static IndexedMatrixValue | toIndexedMatrixBlock(MatrixIndexes ix,
                    MatrixBlock mb) | 
| static IndexedMatrixValue | toIndexedMatrixBlock(scala.Tuple2<MatrixIndexes,MatrixBlock> in) | 
| static IndexedTensorBlock | toIndexedTensorBlock(TensorIndexes ix,
                    TensorBlock mb) | 
| static IndexedTensorBlock | toIndexedTensorBlock(scala.Tuple2<TensorIndexes,TensorBlock> in) | 
public static ExecutorService triggerRDDPool
public static final org.apache.spark.storage.StorageLevel DEFAULT_TMP
public static IndexedMatrixValue toIndexedMatrixBlock(scala.Tuple2<MatrixIndexes,MatrixBlock> in)
public static IndexedMatrixValue toIndexedMatrixBlock(MatrixIndexes ix, MatrixBlock mb)
public static IndexedTensorBlock toIndexedTensorBlock(scala.Tuple2<TensorIndexes,TensorBlock> in)
public static IndexedTensorBlock toIndexedTensorBlock(TensorIndexes ix, TensorBlock mb)
public static scala.Tuple2<MatrixIndexes,MatrixBlock> fromIndexedMatrixBlock(IndexedMatrixValue in)
public static List<scala.Tuple2<MatrixIndexes,MatrixBlock>> fromIndexedMatrixBlock(List<IndexedMatrixValue> in)
public static Pair<MatrixIndexes,MatrixBlock> fromIndexedMatrixBlockToPair(IndexedMatrixValue in)
public static List<Pair<MatrixIndexes,MatrixBlock>> fromIndexedMatrixBlockToPair(List<IndexedMatrixValue> in)
public static scala.Tuple2<Long,FrameBlock> fromIndexedFrameBlock(Pair<Long,FrameBlock> in)
public static List<scala.Tuple2<Long,FrameBlock>> fromIndexedFrameBlock(List<Pair<Long,FrameBlock>> in)
public static Pair<Long,FrameBlock> toIndexedFrameBlock(scala.Tuple2<Long,FrameBlock> in)
public static boolean isHashPartitioned(org.apache.spark.api.java.JavaPairRDD<?,?> in)
org.apache.spark.HashPartitioner.in - input JavaPairRDDpublic static int getNumPreferredPartitions(DataCharacteristics dc, org.apache.spark.api.java.JavaPairRDD<?,?> in)
public static int getNumPreferredPartitions(DataCharacteristics dc)
public static int getNumPreferredPartitions(DataCharacteristics dc, boolean outputEmptyBlocks)
public static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> copyBinaryBlockMatrix(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> in)
in - matrix as JavaPairRDD<MatrixIndexes,MatrixBlock>JavaPairRDD<MatrixIndexes,MatrixBlock>public static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> copyBinaryBlockMatrix(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> in, boolean deep)
in - matrix as JavaPairRDD<MatrixIndexes,MatrixBlock>deep - if true, perform deep copyJavaPairRDD<MatrixIndexes,MatrixBlock>public static org.apache.spark.api.java.JavaPairRDD<TensorIndexes,BasicTensorBlock> copyBinaryBlockTensor(org.apache.spark.api.java.JavaPairRDD<TensorIndexes,BasicTensorBlock> in)
in - tensor as JavaPairRDD<TensorIndexes,HomogTensor>JavaPairRDD<TensorIndexes,HomogTensor>public static org.apache.spark.api.java.JavaPairRDD<TensorIndexes,BasicTensorBlock> copyBinaryBlockTensor(org.apache.spark.api.java.JavaPairRDD<TensorIndexes,BasicTensorBlock> in, boolean deep)
in - tensor as JavaPairRDD<TensorIndexes,HomogTensor>deep - if true, perform deep copyJavaPairRDD<TensorIndexes,HomogTensor>public static void checkSparsity(String varname, ExecutionContext ec)
public static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> getEmptyBlockRDD(org.apache.spark.api.java.JavaSparkContext sc, DataCharacteristics mc)
sc - spark contextmc - matrix characteristicspublic static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> cacheBinaryCellRDD(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> input)
public static DataCharacteristics computeDataCharacteristics(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> input)
input - matrix as JavaPairRDD<MatrixIndexes, MatrixCell>public static long getNonZeros(MatrixObject mo)
public static long getNonZeros(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> input)
public static void postprocessUltraSparseOutput(MatrixObject mo, DataCharacteristics mcOut)
public static void shutdownPool()
Copyright © 2021 The Apache Software Foundation. All rights reserved.