o

com.nvidia.spark.rapids.shims

SparkShimImpl

object SparkShimImpl extends Spark320PlusShims with Spark320PlusNonDBShims with Spark31Xuntil33XShims with Spark320until340Shims

Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. SparkShimImpl
  2. Spark320until340Shims
  3. Spark31Xuntil33XShims
  4. Spark320PlusNonDBShims
  5. Spark320PlusShims
  6. Logging
  7. RebaseShims
  8. SparkShims
  9. AnyRef
  10. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. def ansiCastRule: ExprRule[_ <: Expression]

    Return the replacement rule for AnsiCast.

    Return the replacement rule for AnsiCast. 'AnsiCast' is removed from Spark 3.4.0, so need to handle it separately.

    Definition Classes
    Spark320until340Shims → SparkShims
  5. final def aqeShuffleReaderExec: ExecRule[_ <: SparkPlan]
    Definition Classes
    Spark320PlusShims → SparkShims
  6. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  7. def attachTreeIfSupported[TreeType <: TreeNode[_], A](tree: TreeType, msg: String)(f: ⇒ A): A

    dropped by SPARK-34234

    dropped by SPARK-34234

    Definition Classes
    Spark320PlusShims → SparkShims
  8. final def avroRebaseReadKey: String
    Definition Classes
    RebaseShims
  9. final def avroRebaseWriteKey: String
    Definition Classes
    RebaseShims
  10. final def broadcastModeTransform(mode: BroadcastMode, rows: Array[InternalRow]): Any
    Definition Classes
    Spark320PlusNonDBShims → SparkShims
  11. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native() @HotSpotIntrinsicCandidate()
  12. def columnarAdaptivePlan(a: AdaptiveSparkPlanExec, goal: CoalesceSizeGoal): SparkPlan
    Definition Classes
    Spark320PlusShims → SparkShims
  13. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  14. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  15. final def filesFromFileIndex(fileIndex: PartitioningAwareFileIndex): Seq[FileStatus]
    Definition Classes
    Spark320PlusNonDBShims → SparkShims
  16. def findOperators(plan: SparkPlan, predicate: (SparkPlan) ⇒ Boolean): Seq[SparkPlan]

    Walk the plan recursively and return a list of operators that match the predicate

    Walk the plan recursively and return a list of operators that match the predicate

    Definition Classes
    Spark320PlusShims → SparkShims
  17. def getAdaptiveInputPlan(adaptivePlan: AdaptiveSparkPlanExec): SparkPlan
    Definition Classes
    Spark320PlusShims → SparkShims
  18. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  19. def getDateFormatter(): DateFormatter
    Definition Classes
    Spark320PlusShims → SparkShims
  20. def getExecs: Map[Class[_ <: SparkPlan], ExecRule[_ <: SparkPlan]]
    Definition Classes
    Spark320PlusShims → SparkShims
  21. def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]]
    Definition Classes
    Spark320PlusShims → SparkShims
  22. def getFileScanRDD(sparkSession: SparkSession, readFunction: (PartitionedFile) ⇒ Iterator[InternalRow], filePartitions: Seq[FilePartition], readDataSchema: StructType, metadataColumns: Seq[AttributeReference]): RDD[InternalRow]
    Definition Classes
    SparkShimImpl → SparkShims
  23. def getParquetFilters(schema: MessageType, pushDownDate: Boolean, pushDownTimestamp: Boolean, pushDownDecimal: Boolean, pushDownStartWith: Boolean, pushDownInFilterThreshold: Int, caseSensitive: Boolean, lookupFileMeta: (String) ⇒ String, dateTimeRebaseModeFromConf: String): ParquetFilters
    Definition Classes
    SparkShimImpl → SparkShims
  24. def getScans: Map[Class[_ <: Scan], ScanRule[_ <: Scan]]
    Definition Classes
    Spark320PlusShims → SparkShims
  25. def getSparkShimVersion: ShimVersion
    Definition Classes
    SparkShimImpl → SparkShims
  26. def getWindowExpressions(winPy: WindowInPandasExec): Seq[NamedExpression]
    Definition Classes
    Spark320PlusNonDBShims
  27. def hasAliasQuoteFix: Boolean
    Definition Classes
    Spark320PlusShims → SparkShims
  28. def hasCastFloatTimestampUpcast: Boolean
    Definition Classes
    Spark320PlusShims → SparkShims
  29. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  30. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  31. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  32. def int96ParquetRebaseRead(conf: SQLConf): String
    Definition Classes
    RebaseShims
  33. def int96ParquetRebaseReadKey: String
    Definition Classes
    RebaseShims
  34. def int96ParquetRebaseWrite(conf: SQLConf): String
    Definition Classes
    RebaseShims
  35. def int96ParquetRebaseWriteKey: String
    Definition Classes
    RebaseShims
  36. final def isAqePlan(p: SparkPlan): Boolean
    Definition Classes
    Spark320PlusShims → SparkShims
  37. def isCastingStringToNegDecimalScaleSupported: Boolean
    Definition Classes
    SparkShimImpl → SparkShims
  38. def isCustomReaderExec(x: SparkPlan): Boolean
    Definition Classes
    Spark320PlusShims → SparkShims
  39. def isEmptyRelation(relation: Any): Boolean
    Definition Classes
    Spark320PlusShims → SparkShims
  40. final def isExchangeOp(plan: SparkPlanMeta[_]): Boolean
    Definition Classes
    Spark320PlusShims → SparkShims
  41. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  42. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  43. def isWindowFunctionExec(plan: SparkPlan): Boolean
    Definition Classes
    Spark320PlusShims → SparkShims
  44. def leafNodeDefaultParallelism(ss: SparkSession): Int
    Definition Classes
    Spark320PlusShims → SparkShims
  45. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  46. def logDebug(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  47. def logDebug(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  48. def logError(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  49. def logError(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  50. def logInfo(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  51. def logInfo(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  52. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  53. def logTrace(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  54. def logTrace(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  55. def logWarning(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  56. def logWarning(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  57. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  58. def neverReplaceShowCurrentNamespaceCommand: ExecRule[_ <: SparkPlan]
    Definition Classes
    Spark31Xuntil33XShims → SparkShims
  59. final def newBroadcastQueryStageExec(old: BroadcastQueryStageExec, newPlan: SparkPlan): BroadcastQueryStageExec
    Definition Classes
    Spark320PlusNonDBShims → SparkShims
  60. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  61. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  62. final def parquetRebaseRead(conf: SQLConf): String
    Definition Classes
    RebaseShims
  63. final def parquetRebaseReadKey: String
    Definition Classes
    RebaseShims
  64. final def parquetRebaseWrite(conf: SQLConf): String
    Definition Classes
    RebaseShims
  65. final def parquetRebaseWriteKey: String
    Definition Classes
    RebaseShims
  66. def reusedExchangeExecPfn: PartialFunction[SparkPlan, ReusedExchangeExec]

    Case class ShuffleQueryStageExec holds an additional field shuffleOrigin affecting the unapply method signature

    Case class ShuffleQueryStageExec holds an additional field shuffleOrigin affecting the unapply method signature

    Definition Classes
    Spark320PlusNonDBShims → SparkShims
  67. final def sessionFromPlan(plan: SparkPlan): SparkSession
    Definition Classes
    Spark320PlusShims → SparkShims
  68. def shouldFailDivOverflow(): Boolean
    Definition Classes
    Spark320PlusShims → SparkShims
  69. def skipAssertIsOnTheGpu(plan: SparkPlan): Boolean

    Our tests, by default, will check that all operators are running on the GPU, but there are some operators that we do not translate to GPU plans, so we need a way to bypass the check for those.

    Our tests, by default, will check that all operators are running on the GPU, but there are some operators that we do not translate to GPU plans, so we need a way to bypass the check for those.

    Definition Classes
    Spark320PlusShims → SparkShims
  70. def supportsColumnarAdaptivePlans: Boolean

    Determine if the Spark version allows the supportsColumnar flag to be overridden in AdaptiveSparkPlanExec.

    Determine if the Spark version allows the supportsColumnar flag to be overridden in AdaptiveSparkPlanExec. This feature was introduced in Spark 3.2 as part of SPARK-35881.

    Definition Classes
    Spark320PlusShims → SparkShims
  71. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  72. def tagFileSourceScanExec(meta: SparkPlanMeta[FileSourceScanExec]): Unit
    Definition Classes
    Spark320PlusShims
  73. def toString(): String
    Definition Classes
    AnyRef → Any
  74. def tryTransformIfEmptyRelation(mode: BroadcastMode): Option[Any]

    This call can produce an EmptyHashedRelation or an empty array, allowing the AQE rule EliminateJoinToEmptyRelation in Spark 3.1.x to optimize certain joins.

    This call can produce an EmptyHashedRelation or an empty array, allowing the AQE rule EliminateJoinToEmptyRelation in Spark 3.1.x to optimize certain joins.

    In Spark 3.2.0, the optimization is still performed (under AQEPropagateEmptyRelation), but the AQE optimizer is looking at the metrics for the query stage to determine if numRows == 0, and if so it can eliminate certain joins.

    The call is implemented only for Spark 3.1.x+. It is disabled in Databricks because it requires a task context to perform the BroadcastMode.transform call, but we'd like to call this from the driver.

    Definition Classes
    Spark320PlusShims → SparkShims
  75. def v1RepairTableCommand(tableName: TableIdentifier): RunnableCommand
    Definition Classes
    Spark320PlusShims → SparkShims
  76. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  77. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  78. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] ) @Deprecated
    Deprecated

Inherited from Spark320until340Shims

Inherited from Spark31Xuntil33XShims

Inherited from Spark320PlusNonDBShims

Inherited from Spark320PlusShims

Inherited from Logging

Inherited from RebaseShims

Inherited from SparkShims

Inherited from AnyRef

Inherited from Any

Ungrouped