Columnar
spark/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala
/**
* Provides a common executor to translate an [[RDD]] of [[ColumnarBatch]] into an [[RDD]] of
* [[InternalRow]]. This is inserted whenever such a transition is determined to be needed.
*
* The implementation is based off of similar implementations in
* [[org.apache.spark.sql.execution.python.ArrowEvalPythonExec]] and
* [[MapPartitionsInRWithArrowExec]]. Eventually this should replace those implementations.
*/
case class ColumnarToRowExec(child: SparkPlan) extends ColumnarToRowTransition with CodegenSupport
/**
* Provides an optimized set of APIs to append row based data to an array of
* [[WritableColumnVector]].
*/
private[execution] class RowToColumnConverter(schema: StructType) extends Serializable
/**
* Provides an optimized set of APIs to extract a column from a row and append it to a
* [[WritableColumnVector]].
*/
private object RowToColumnConverter
/**
* Provides a common executor to translate an [[RDD]] of [[InternalRow]] into an [[RDD]] of
* [[ColumnarBatch]]. This is inserted whenever such a transition is determined to be needed.
*
* This is similar to some of the code in ArrowConverters.scala and
* [[org.apache.spark.sql.execution.arrow.ArrowWriter]]. That code is more specialized
* to convert [[InternalRow]] to Arrow formatted data, but in the future if we make
* [[OffHeapColumnVector]] internally Arrow formatted we may be able to replace much of that code.
*
* This is also similar to
* [[org.apache.spark.sql.execution.vectorized.ColumnVectorUtils.populate()]] and
* [[org.apache.spark.sql.execution.vectorized.ColumnVectorUtils.toBatch()]] toBatch is only ever
* called from tests and can probably be removed, but populate is used by both Orc and Parquet
* to initialize partition and missing columns. There is some chance that we could replace
* populate with [[RowToColumnConverter]], but the performance requirements are different and it
* would only be to reduce code.
*/
case class RowToColumnarExec(child: SparkPlan) extends RowToColumnarTransition
