我是一个AWS胶水新手试图读取我在S3中的一些镶木地板对象,但我失败了ClassNotFoundException.这是我到目前为止基于Glue的最小文档的尝试:
import com.amazonaws.services.glue.GlueContext
import com.amazonaws.services.glue.util.JsonOptions
import org.apache.spark.sql.SparkSession
val gc: GlueContext = new GlueContext(sc)
val spark_session : SparkSession = gc.getSparkSession
val source = gc.getSource("s3", JsonOptions(Map("paths" -> Set("s3://path-to-parquet"))))
val parquetSource = source.withFormat("parquet")
parquetSource.getDynamicFrame().show(1)
例外情况:
18/06/11 13:39:11 WARN TaskSetManager: Lost task 0.0 in stage 6.0 (TID 266, ip-172-31-8-179.eu-west-1.compute.internal, executor 16): java.lang.ClassNotFoundException: Failed to load format with name parquet
at com.amazonaws.services.glue.util.ClassUtils$.loadByFullName(ClassUtils.scala:28)
at com.amazonaws.services.glue.util.ClassUtils$.getClassByName(ClassUtils.scala:43)
at com.amazonaws.services.glue.util.ClassUtils$.newInstanceByName(ClassUtils.scala:54)
at com.amazonaws.services.glue.readers.DynamicRecordStreamReader$.apply(DynamicRecordReader.scala:187)
...
Caused by: java.lang.ClassNotFoundException: parquet
at org.apache.spark.repl.ExecutorClassLoader.findClass(ExecutorClassLoader.scala:82)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at com.amazonaws.services.glue.util.ClassUtils$$anonfun$1.apply(ClassUtils.scala:25)
at com.amazonaws.services.glue.util.ClassUtils$$anonfun$1.apply(ClassUtils.scala:25)
at scala.util.Try$.apply(Try.scala:192) …