小编Rox*_*ana的帖子

我有以下代码:

val df_in = sqlcontext.read.json(jsonFile) // the file resides in hdfs

//some operations in here to create df as df_in with two more columns "terms1" and "terms2" 

val intersectUDF = udf( (seq1:Seq[String], seq2:Seq[String] ) => {     seq1 intersect seq2 } ) //intersects two sequences
val symmDiffUDF = udf( (seq1:Seq[String], seq2:Seq[String] ) => { (seq1 diff seq2) ++ (seq2 diff seq1) } ) //compute the difference of two sequences

val df1 = (df.withColumn("termsInt", intersectUDF(df("terms1"), df1("terms2") ) )
             .withColumn("termsDiff", symmDiffUDF(df("terms1"),     df1("terms2") ) …