class CSVPartitionReader extends GpuTextBasedPartitionReader
- Alphabetic
- By Inheritance
- CSVPartitionReader
- GpuTextBasedPartitionReader
- Arm
- ScanWithMetrics
- PartitionReader
- Closeable
- AutoCloseable
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
- new CSVPartitionReader(conf: Configuration, partFile: PartitionedFile, dataSchema: StructType, readDataSchema: StructType, parsedOptions: CSVOptions, maxRowsPerChunk: Integer, maxBytesPerChunk: Long, execMetrics: Map[String, GpuMetric])
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
- def buildCsvOptions(parsedOptions: CSVOptions, schema: StructType, hasHeader: Boolean): CSVOptions
-
def
castStringToBool(input: ColumnVector): ColumnVector
CSV supports "true" and "false" (case-insensitive) as valid boolean values.
CSV supports "true" and "false" (case-insensitive) as valid boolean values.
- Definition Classes
- CSVPartitionReader → GpuTextBasedPartitionReader
-
def
castStringToDate(input: ColumnVector, dt: DType, failOnInvalid: Boolean): ColumnVector
- Definition Classes
- GpuTextBasedPartitionReader
-
def
castStringToDecimal(input: ColumnVector, dt: DecimalType): ColumnVector
- Definition Classes
- GpuTextBasedPartitionReader
-
def
castStringToFloat(input: ColumnVector, dt: DType): ColumnVector
- Definition Classes
- GpuTextBasedPartitionReader
-
def
castStringToInt(input: ColumnVector, intType: DType): ColumnVector
- Definition Classes
- GpuTextBasedPartitionReader
-
def
castStringToTimestamp(lhs: ColumnVector, sparkFormat: String, dtype: DType): ColumnVector
- Definition Classes
- GpuTextBasedPartitionReader
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native() @HotSpotIntrinsicCandidate()
-
def
close(): Unit
- Definition Classes
- GpuTextBasedPartitionReader → Closeable → AutoCloseable
-
def
closeOnExcept[T <: AutoCloseable, V](r: Option[T])(block: (Option[T]) ⇒ V): V
Executes the provided code block, closing the resources only if an exception occurs
Executes the provided code block, closing the resources only if an exception occurs
- Definition Classes
- Arm
-
def
closeOnExcept[T <: AutoCloseable, V](r: ArrayBuffer[T])(block: (ArrayBuffer[T]) ⇒ V): V
Executes the provided code block, closing the resources only if an exception occurs
Executes the provided code block, closing the resources only if an exception occurs
- Definition Classes
- Arm
-
def
closeOnExcept[T <: AutoCloseable, V](r: Array[T])(block: (Array[T]) ⇒ V): V
Executes the provided code block, closing the resources only if an exception occurs
Executes the provided code block, closing the resources only if an exception occurs
- Definition Classes
- Arm
-
def
closeOnExcept[T <: AutoCloseable, V](r: Seq[T])(block: (Seq[T]) ⇒ V): V
Executes the provided code block, closing the resources only if an exception occurs
Executes the provided code block, closing the resources only if an exception occurs
- Definition Classes
- Arm
-
def
closeOnExcept[T <: AutoCloseable, V](r: T)(block: (T) ⇒ V): V
Executes the provided code block, closing the resource only if an exception occurs
Executes the provided code block, closing the resource only if an exception occurs
- Definition Classes
- Arm
-
def
currentMetricsValues(): Array[CustomTaskMetric]
- Definition Classes
- PartitionReader
-
def
dateFormat: String
- Definition Classes
- CSVPartitionReader → GpuTextBasedPartitionReader
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
freeOnExcept[T <: RapidsBuffer, V](r: T)(block: (T) ⇒ V): V
Executes the provided code block, freeing the RapidsBuffer only if an exception occurs
Executes the provided code block, freeing the RapidsBuffer only if an exception occurs
- Definition Classes
- Arm
-
def
get(): ColumnarBatch
- Definition Classes
- GpuTextBasedPartitionReader → PartitionReader
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native() @HotSpotIntrinsicCandidate()
-
def
getFileFormatShortName: String
File format short name used for logging and other things to uniquely identity which file format is being used.
File format short name used for logging and other things to uniquely identity which file format is being used.
- returns
the file format short name
- Definition Classes
- CSVPartitionReader → GpuTextBasedPartitionReader
-
def
handleResult(readDataSchema: StructType, table: Table): Option[Table]
Handle the table decoded by GPU
Handle the table decoded by GPU
Please note that, this function owns table which is supposed to be closed in this function But for the optimization, we just return the original table.
- readDataSchema
the Spark schema describing what will be read
- table
the table decoded by GPU
- returns
the new optional Table
- Definition Classes
- GpuTextBasedPartitionReader
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native() @HotSpotIntrinsicCandidate()
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
val
metrics: Map[String, GpuMetric]
- Definition Classes
- ScanWithMetrics
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
next(): Boolean
- Definition Classes
- GpuTextBasedPartitionReader → PartitionReader
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native() @HotSpotIntrinsicCandidate()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native() @HotSpotIntrinsicCandidate()
-
def
readToTable(dataBuffer: HostMemoryBuffer, dataSize: Long, cudfSchema: Schema, readDataSchema: StructType, isFirstChunk: Boolean): Table
Read the host buffer to GPU table
Read the host buffer to GPU table
- dataBuffer
host buffer to be read
- dataSize
the size of host buffer
- cudfSchema
the cudf schema of the data
- readDataSchema
the Spark schema describing what will be read
- isFirstChunk
if it is the first chunk
- returns
table
- Definition Classes
- CSVPartitionReader → GpuTextBasedPartitionReader
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
timestampFormat: String
- Definition Classes
- CSVPartitionReader → GpuTextBasedPartitionReader
-
def
toString(): String
- Definition Classes
- AnyRef → Any
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
def
withResource[T <: AutoCloseable, V](h: CloseableHolder[T])(block: (CloseableHolder[T]) ⇒ V): V
Executes the provided code block and then closes the resource
Executes the provided code block and then closes the resource
- Definition Classes
- Arm
-
def
withResource[T <: AutoCloseable, V](r: ArrayBuffer[T])(block: (ArrayBuffer[T]) ⇒ V): V
Executes the provided code block and then closes the array buffer of resources
Executes the provided code block and then closes the array buffer of resources
- Definition Classes
- Arm
-
def
withResource[T <: AutoCloseable, V](r: Array[T])(block: (Array[T]) ⇒ V): V
Executes the provided code block and then closes the array of resources
Executes the provided code block and then closes the array of resources
- Definition Classes
- Arm
-
def
withResource[T <: AutoCloseable, V](r: Seq[T])(block: (Seq[T]) ⇒ V): V
Executes the provided code block and then closes the sequence of resources
Executes the provided code block and then closes the sequence of resources
- Definition Classes
- Arm
-
def
withResource[T <: AutoCloseable, V](r: Option[T])(block: (Option[T]) ⇒ V): V
Executes the provided code block and then closes the Option[resource]
Executes the provided code block and then closes the Option[resource]
- Definition Classes
- Arm
-
def
withResource[T <: AutoCloseable, V](r: T)(block: (T) ⇒ V): V
Executes the provided code block and then closes the resource
Executes the provided code block and then closes the resource
- Definition Classes
- Arm
-
def
withResourceIfAllowed[T, V](r: T)(block: (T) ⇒ V): V
Executes the provided code block and then closes the value if it is AutoCloseable
Executes the provided code block and then closes the value if it is AutoCloseable
- Definition Classes
- Arm