从CSV创建DataFrame

示例

CSV内容

name,age
alice,18
bob,14
cassie,

scala代码

import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.types.{StringType, IntegerType, StructField, StructType}

// 定义schema
val schema:StructType = StructType( Array(
StructField("name", StringType),
StructField("age", IntegerType)
))

// CSV文件目录
val csvFilePath: String = _

// 创建DataFrame
val df: DataFrame = spark.read.format("csv").schema(schema).option("header", true).option("mode", "dropMalformed").load(csvFilePath)
// df: org.apache.spark.sql.DataFrame = [name: string, age: int]
df.show

option参数