小编Ade*_*apo的帖子

我正在使用“spark-submit --master local --executor-memory 800m target/scala-2.10/finalproject_2.10-1.0.jar”提交作业

我的 sbt 文件具有以下依赖项：

name := "Projectx"

version := "1.0"

scalaVersion := "2.10.6"

libraryDependencies += "org.apache.spark" %% "spark-core" % "1.6.3"

libraryDependencies += "org.apache.spark" % "spark-sql_2.10" % "1.6.1"

libraryDependencies += "com.databricks" % "spark-csv_2.10" % "1.4.0"

我的代码：

object Projectx {
  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("AirlineAnalysis")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)
     sc.setLogLevel("ERROR")


    // HDFS
    val rawFlights= sc.textFile("hdfs://localhost:9000/Project/2008.csv")
       rawFlights.take(5) 
 val df = sqlContext.read.format("com.databricks.spark.csv").option("header", "true").load("hdfs://localhost:9000/Project/2008.csv")
    df.take(5)
    df.printSchema()
    df.col("Year").cast("int")
   val df_1 = …