数据转换GraphFrame 效率太低了,connectedComponents的算法效率惨不忍睹,必须要设置graphx算法。
代码如下,
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.graphframes.GraphFrame;
/**
*
*/
public class MiniTreeGraphFrame
{
private static SQLContext sqlCtx;
private static StructType eType;
public static void main( String[] args )
{
SparkConf conf = new SparkConf( ).setAppName( "Short Paths" ).setMaster( "local" );
JavaSparkContext ctx = new JavaSparkContext( conf );
sqlCtx = SQLContext.getOrCreate( ctx.sc( ) );
List<StructField> vList = new ArrayList<StructField>( );
vList.add( DataTypes.createStructField( "id", DataTypes.LongType, false ) );
vList.add( DataTypes.createStructField( "name", DataTypes.StringType, true ) );
StructType vType = DataTypes.createStructType( vList );