我有一个简单的火花功能来测试DF窗口:
import org.apache.spark.sql.{DataFrame, SparkSession}
object ScratchPad {
def main(args: Array[String]): Unit = {
val spark = SparkSession.builder().master("local[*]").getOrCreate()
spark.sparkContext.setLogLevel("ERROR")
get_data_frame(spark).show()
}
def get_data_frame(spark: SparkSession): DataFrame = {
import spark.sqlContext.implicits._
val hr = spark.sparkContext.parallelize(List(
("Steinbeck", "Sales", 100),
("Woolf", "IT", 99),
("Wodehouse", "Sales", 250),
("Hemingway", "IT", 349)
)
).toDF("emp", "dept", "sal")
import org.apache.spark.sql.expressions.Window
import org.apache.spark.sql.functions._
val windowspec = Window.partitionBy($"dept").orderBy($"sal".desc)
hr.withColumn("rank", row_number().over(windowspec))
}
}
Run Code Online (Sandbox Code Playgroud)
我写了一个像这样的测试:
import com.holdenkarau.spark.testing.DataFrameSuiteBase
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
import org.scalatest.FunSuite
class TestDF extends FunSuite with DataFrameSuiteBase {
test ("DFs equal") { …Run Code Online (Sandbox Code Playgroud) 我在一本java书中发现了这个,并想了解为什么我会收到如下所述的错误:
public class Wildcard<V> {
V value;
public Wildcard(V value) {
super();
this.value = value;
}
public V getValue() {
return value;
}
public void setValue(V value) {
this.value = value;
}
public boolean sameObj(Wildcard<? extends V> v1, MyComparator<V> comparator) {
return comparator.compare(this.value, v1.value);
}
public static void main(String[] args) {
Wildcard<Integer> iTest = new Wildcard<Integer>(10);
Wildcard<Number> nTest = new Wildcard<Number>(10);
nTest.sameObject(iTest, new MyComparator<Number>() {
public boolean compare(Number o1, Number o2) {
return o1 == o2;
}
});
}
} …Run Code Online (Sandbox Code Playgroud)