Packages

class SQLConf extends Serializable with Logging

A class that enables the setting and getting of mutable config parameters/hints.

In the presence of a SQLContext, these can be set and queried by passing SET commands into Spark SQL's query functions (i.e. sql()). Otherwise, users of this class can modify the hints by programmatically calling the setters and getters of this class.

SQLConf is thread-safe (internally synchronized, so safe to be used in multiple threads).

Linear Supertypes
Logging, Serializable, Serializable, AnyRef, Any
Known Subclasses
ReadOnlySQLConf
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. SQLConf
  2. Logging
  3. Serializable
  4. Serializable
  5. AnyRef
  6. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new SQLConf()

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. def adaptiveExecutionEnabled: Boolean
  5. def adaptiveExecutionLogLevel: String
  6. def addSingleFileInAddFile: Boolean
  7. def advancedPartitionPredicatePushdownEnabled: Boolean
  8. def allowAutoGeneratedAliasForView: Boolean
  9. def allowNegativeScaleOfDecimalEnabled: Boolean
  10. def allowNonEmptyLocationInCTAS: Boolean
  11. def allowStarWithSingleTableIdentifierInCount: Boolean
  12. def analyzerMaxIterations: Int

    ************************ Spark SQL Params/Hints *******************

  13. def ansiEnabled: Boolean
  14. def arrowMaxRecordsPerBatch: Int
  15. def arrowPySparkEnabled: Boolean
  16. def arrowPySparkFallbackEnabled: Boolean
  17. def arrowPySparkSelfDestructEnabled: Boolean
  18. def arrowSafeTypeConversion: Boolean
  19. def arrowSparkREnabled: Boolean
  20. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  21. def autoBroadcastJoinThreshold: Long
  22. def autoBucketedScanEnabled: Boolean
  23. def autoSizeUpdateEnabled: Boolean
  24. def avroCompressionCodec: String
  25. def avroDeflateLevel: Int
  26. def avroFilterPushDown: Boolean
  27. def broadcastHashJoinOutputPartitioningExpandLimit: Int
  28. def broadcastTimeout: Long
  29. def bucketingEnabled: Boolean
  30. def bucketingMaxBuckets: Int
  31. def cacheVectorizedReaderEnabled: Boolean
  32. def cartesianProductExecBufferInMemoryThreshold: Int
  33. def cartesianProductExecBufferSpillThreshold: Int
  34. def caseSensitiveAnalysis: Boolean
  35. def caseSensitiveInferenceMode: SQLConf.HiveCaseSensitiveInferenceMode.Value
  36. def castDatetimeToString: Boolean
  37. def cboEnabled: Boolean
  38. def charVarcharAsString: Boolean
  39. def checkpointLocation: Option[String]
  40. def clear(): Unit
  41. def cliPrintHeader: Boolean
  42. def clone(): SQLConf
    Definition Classes
    SQLConf → AnyRef
  43. def coalesceBucketsInJoinEnabled: Boolean
  44. def coalesceBucketsInJoinMaxBucketRatio: Int
  45. def coalesceShufflePartitionsEnabled: Boolean
  46. def codegenCacheMaxEntries: Int
  47. def codegenComments: Boolean
  48. def codegenFallback: Boolean
  49. def codegenSplitAggregateFunc: Boolean
  50. def columnBatchSize: Int
  51. def columnNameOfCorruptRecord: String
  52. def concatBinaryAsString: Boolean
  53. def constraintPropagationEnabled: Boolean
  54. def contains(key: String): Boolean

    Return whether a given key is set in this SQLConf.

  55. def continuousStreamingEpochBacklogQueueSize: Int
  56. def continuousStreamingExecutorPollIntervalMs: Long
  57. def continuousStreamingExecutorQueueSize: Int
  58. def convertCTAS: Boolean
  59. def copy(entries: (ConfigEntry[_], Any)*): SQLConf
  60. def crossJoinEnabled: Boolean
  61. def csvColumnPruning: Boolean
  62. def csvExpressionOptimization: Boolean
  63. def csvFilterPushDown: Boolean
  64. def dataFramePivotMaxValues: Int
  65. def dataFrameRetainGroupColumns: Boolean
  66. def dataFrameSelfJoinAutoResolveAmbiguity: Boolean
  67. def datetimeJava8ApiEnabled: Boolean
  68. def decimalOperationsAllowPrecisionLoss: Boolean
  69. def decorrelateInnerQueryEnabled: Boolean
  70. def defaultDataSourceName: String
  71. def defaultNumShufflePartitions: Int
  72. def defaultSizeInBytes: Long
  73. def disabledJdbcConnectionProviders: String
  74. def disabledV2StreamingMicroBatchReaders: String
  75. def disabledV2StreamingWriters: String
  76. def dynamicPartitionPruningEnabled: Boolean
  77. def dynamicPartitionPruningFallbackFilterRatio: Double
  78. def dynamicPartitionPruningReuseBroadcastOnly: Boolean
  79. def dynamicPartitionPruningUseStats: Boolean
  80. def eltOutputAsString: Boolean
  81. def enableRadixSort: Boolean
  82. def enableTwoLevelAggMap: Boolean
  83. def enableVectorizedHashMap: Boolean
  84. def enforceReservedKeywords: Boolean
  85. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  86. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  87. def escapedStringLiterals: Boolean
  88. def exchangeReuseEnabled: Boolean
  89. def exponentLiteralAsDecimalEnabled: Boolean
  90. def fallBackToHdfsForStatsEnabled: Boolean
  91. def fastFailFileFormatOutput: Boolean
  92. def fastHashAggregateRowMaxCapacityBit: Int
  93. def fetchShuffleBlocksInBatch: Boolean
  94. def fileCommitProtocolClass: String
  95. def fileCompressionFactor: Double
  96. def fileSinkLogCleanupDelay: Long
  97. def fileSinkLogCompactInterval: Int
  98. def fileSinkLogDeletion: Boolean
  99. def fileSourceLogCleanupDelay: Long
  100. def fileSourceLogCompactInterval: Int
  101. def fileSourceLogDeletion: Boolean
  102. def fileStreamSinkMetadataIgnored: Boolean
  103. def filesMaxPartitionBytes: Long
  104. def filesMinPartitionNum: Option[Int]
  105. def filesOpenCostInBytes: Long
  106. def filesourcePartitionFileCacheSize: Long
  107. def gatherFastStats: Boolean
  108. def getAllConfs: Map[String, String]

    Return all the configuration properties that have been set (i.e.

    Return all the configuration properties that have been set (i.e. not the default). This creates a new copy of the config properties in the form of a Map.

  109. def getAllDefinedConfs: Seq[(String, String, String, String)]

    Return all the configuration definitions that have been defined in SQLConf.

    Return all the configuration definitions that have been defined in SQLConf. Each definition contains key, defaultValue and doc.

  110. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  111. def getConf[T](entry: OptionalConfigEntry[T]): Option[T]

    Return the value of an optional Spark SQL configuration property for the given key.

    Return the value of an optional Spark SQL configuration property for the given key. If the key is not set yet, returns None.

  112. def getConf[T](entry: ConfigEntry[T]): T

    Return the value of Spark SQL configuration property for the given key.

    Return the value of Spark SQL configuration property for the given key. If the key is not set yet, return defaultValue in ConfigEntry.

  113. def getConf[T](entry: ConfigEntry[T], defaultValue: T): T

    Return the value of Spark SQL configuration property for the given key.

    Return the value of Spark SQL configuration property for the given key. If the key is not set yet, return defaultValue. This is useful when defaultValue in ConfigEntry is not the desired one.

  114. def getConfString(key: String, defaultValue: String): String

    Return the string value of Spark SQL configuration property for the given key.

    Return the string value of Spark SQL configuration property for the given key. If the key is not set yet, return defaultValue.

  115. def getConfString(key: String): String

    Return the value of Spark SQL configuration property for the given key.

    Return the value of Spark SQL configuration property for the given key.

    Annotations
    @throws( "if key is not set" )
  116. def groupByAliases: Boolean
  117. def groupByOrdinal: Boolean
  118. def groupingIdWithAppendedUserGroupByEnabled: Boolean
  119. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  120. def hintErrorHandler: HintErrorHandler

    Returns the error handler for handling hint errors.

  121. def histogramEnabled: Boolean
  122. def histogramNumBins: Int
  123. def histogramNumericPropagateInputType: Boolean
  124. def hiveThriftServerSingleSession: Boolean
  125. def hugeMethodLimit: Int
  126. def ignoreCorruptFiles: Boolean
  127. def ignoreDataLocality: Boolean
  128. def ignoreMissingFiles: Boolean
  129. def ignoreMissingParquetFieldId: Boolean
  130. def inMemoryPartitionPruning: Boolean
  131. def inMemoryTableScanStatisticsEnabled: Boolean
  132. def inferDictAsStruct: Boolean
  133. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  134. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  135. def integerGroupingIdEnabled: Boolean
  136. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  137. def isModifiable(key: String): Boolean
  138. def isOrcSchemaMergingEnabled: Boolean
  139. def isParquetBinaryAsString: Boolean
  140. def isParquetINT96AsTimestamp: Boolean
  141. def isParquetINT96TimestampConversion: Boolean
  142. def isParquetSchemaMergingEnabled: Boolean
  143. def isParquetSchemaRespectSummaries: Boolean
  144. def isReplEagerEvalEnabled: Boolean
  145. def isStateSchemaCheckEnabled: Boolean
  146. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  147. def isUnsupportedOperationCheckEnabled: Boolean
  148. def joinReorderCardWeight: Double
  149. def joinReorderDPStarFilter: Boolean
  150. def joinReorderDPThreshold: Int
  151. def joinReorderEnabled: Boolean
  152. def jsonExpressionOptimization: Boolean
  153. def jsonFilterPushDown: Boolean
  154. def jsonGeneratorIgnoreNullFields: Boolean
  155. def legacyIntervalEnabled: Boolean
  156. def legacyMsSqlServerNumericMappingEnabled: Boolean
  157. def legacyParquetNanosAsLong: Boolean
  158. def legacyPathOptionBehavior: Boolean
  159. def legacySizeOfNull: Boolean
  160. def legacyStatisticalAggregate: Boolean
  161. def legacyTimeParserPolicy: SQLConf.LegacyBehaviorPolicy.Value
  162. def limitScaleUpFactor: Int
  163. def literalPickMinimumPrecision: Boolean
  164. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  165. def logDebug(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  166. def logDebug(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  167. def logError(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  168. def logError(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  169. def logInfo(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  170. def logInfo(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  171. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  172. def logTrace(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  173. def logTrace(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  174. def logWarning(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  175. def logWarning(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  176. def loggingMaxLinesForCodegen: Int
  177. def manageFilesourcePartitions: Boolean
  178. def maxBatchesToRetainInMemory: Int
  179. def maxConcurrentOutputFileWriters: Int
  180. def maxMetadataStringLength: Int
  181. def maxNestedViewDepth: Int
  182. def maxPlanStringLength: Int
  183. def maxRecordsPerFile: Long
  184. def maxToStringFields: Int
  185. def metadataCacheTTL: Long
  186. def metastorePartitionPruning: Boolean
  187. def metastorePartitionPruningFallbackOnException: Boolean
  188. def metastorePartitionPruningFastFallback: Boolean
  189. def metastorePartitionPruningInSetThreshold: Int
  190. def methodSplitThreshold: Int
  191. def minBatchesToRetain: Int
  192. def nameNonStructGroupingKeyAsValue: Boolean
  193. def ndvMaxError: Double
  194. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  195. def nestedPruningOnExpressions: Boolean
  196. def nestedSchemaPruningEnabled: Boolean
  197. def nonEmptyPartitionRatioForBroadcastJoin: Double
  198. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  199. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  200. def numShufflePartitions: Int
  201. def objectAggSortBasedFallbackThreshold: Int
  202. def offHeapColumnVectorEnabled: Boolean
  203. def optimizeNullAwareAntiJoin: Boolean
  204. def optimizerExcludedRules: Option[String]
  205. def optimizerInSetConversionThreshold: Int
  206. def optimizerInSetSwitchThreshold: Int
  207. def optimizerMaxIterations: Int
  208. def optimizerMetadataOnly: Boolean
  209. def orcAggregatePushDown: Boolean
  210. def orcCompressionCodec: String
  211. def orcFilterPushDown: Boolean
  212. def orcVectorizedReaderBatchSize: Int
  213. def orcVectorizedReaderEnabled: Boolean
  214. def orcVectorizedReaderNestedColumnEnabled: Boolean
  215. def orderByOrdinal: Boolean
  216. def pandasGroupedMapAssignColumnsByName: Boolean
  217. def pandasUDFBufferSize: Int
  218. def parallelFileListingInStatsComputation: Boolean
  219. def parallelPartitionDiscoveryParallelism: Int
  220. def parallelPartitionDiscoveryThreshold: Int
  221. def parquetAggregatePushDown: Boolean
  222. def parquetCompressionCodec: String
  223. def parquetFieldIdReadEnabled: Boolean
  224. def parquetFieldIdWriteEnabled: Boolean
  225. def parquetFilterPushDown: Boolean
  226. def parquetFilterPushDownDate: Boolean
  227. def parquetFilterPushDownDecimal: Boolean
  228. def parquetFilterPushDownInFilterThreshold: Int
  229. def parquetFilterPushDownStringStartWith: Boolean
  230. def parquetFilterPushDownTimestamp: Boolean
  231. def parquetOutputCommitterClass: String
  232. def parquetOutputTimestampType: SQLConf.ParquetOutputTimestampType.Value
  233. def parquetRecordFilterEnabled: Boolean
  234. def parquetVectorizedReaderBatchSize: Int
  235. def parquetVectorizedReaderEnabled: Boolean
  236. def parquetVectorizedReaderNestedColumnEnabled: Boolean
  237. def partitionColumnTypeInferenceEnabled: Boolean
  238. def partitionOverwriteMode: SQLConf.PartitionOverwriteMode.Value
  239. def percentileAccuracy: Int
  240. def planChangeBatches: Option[String]
  241. def planChangeLogLevel: String
  242. def planChangeRules: Option[String]
  243. def planStatsEnabled: Boolean
  244. def preferSortMergeJoin: Boolean
  245. def pysparkJVMStacktraceEnabled: Boolean
  246. def pysparkSimplifiedTraceback: Boolean
  247. def rangeExchangeSampleSizePerPartition: Int
  248. val reader: ConfigReader
    Attributes
    protected
  249. def redactOptions[K, V](options: Seq[(K, V)]): Seq[(K, V)]

    Redacts the given option map according to the description of SQL_OPTIONS_REDACTION_PATTERN.

  250. def redactOptions[K, V](options: Map[K, V]): Map[K, V]

    Redacts the given option map according to the description of SQL_OPTIONS_REDACTION_PATTERN.

  251. def replEagerEvalMaxNumRows: Int
  252. def replEagerEvalTruncate: Int
  253. def replaceDatabricksSparkAvroEnabled: Boolean
  254. def replaceExceptWithFilter: Boolean
  255. def resolver: Resolver

    Returns the Resolver for the current configuration, which can be used to determine if two identifiers are equal.

  256. def runSQLonFile: Boolean
  257. def runtimeFilterBloomFilterEnabled: Boolean
  258. def runtimeFilterCreationSideThreshold: Long
  259. def runtimeFilterSemiJoinReductionEnabled: Boolean
  260. def serializerNestedSchemaPruningEnabled: Boolean
  261. def sessionLocalTimeZone: String
  262. def sessionWindowBufferInMemoryThreshold: Int
  263. def sessionWindowBufferSpillThreshold: Int
  264. def setCommandRejectsSparkCoreConfs: Boolean
  265. def setConf[T](entry: ConfigEntry[T], value: T): Unit

    Set the given Spark SQL configuration property.

  266. def setConf(props: Properties): Unit

    Set Spark SQL configuration properties.

  267. def setConfString(key: String, value: String): Unit

    Set the given Spark SQL configuration property using a string value.

  268. def setConfWithCheck(key: String, value: String): Unit
    Attributes
    protected
  269. def setOpsPrecedenceEnforced: Boolean
  270. val settings: Map[String, String]

    Only low degree of contention is expected for conf, thus NOT using ConcurrentHashMap.

    Only low degree of contention is expected for conf, thus NOT using ConcurrentHashMap.

    Attributes
    protected[spark]
  271. def sortBeforeRepartition: Boolean
  272. def sortMergeJoinExecBufferInMemoryThreshold: Int
  273. def sortMergeJoinExecBufferSpillThreshold: Int
  274. def starSchemaDetection: Boolean
  275. def starSchemaFTRatio: Double
  276. def stateStoreCompressionCodec: String
  277. def stateStoreFormatValidationEnabled: Boolean
  278. def stateStoreMinDeltasForSnapshot: Int
  279. def stateStoreProviderClass: String
  280. def stateStoreSkipNullsForStreamStreamJoins: Boolean
  281. def statefulOperatorCorrectnessCheckEnabled: Boolean
  282. def storeAnalyzedPlanForView: Boolean
  283. def storeAssignmentPolicy: SQLConf.StoreAssignmentPolicy.Value
  284. def streamingFileCommitProtocolClass: String
  285. def streamingMaintenanceInterval: Long
  286. def streamingMetricsEnabled: Boolean
  287. def streamingNoDataMicroBatchesEnabled: Boolean
  288. def streamingNoDataProgressEventInterval: Long
  289. def streamingPollingDelay: Long
  290. def streamingProgressRetention: Int
  291. def streamingSchemaInference: Boolean
  292. def streamingSessionWindowMergeSessionInLocalPartition: Boolean
  293. def strictIndexOperator: Boolean
  294. def stringRedactionPattern: Option[Regex]
  295. def subexpressionEliminationCacheMaxEntries: Int
  296. def subexpressionEliminationEnabled: Boolean
  297. def subqueryReuseEnabled: Boolean
  298. def supportQuotedRegexColumnName: Boolean
  299. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  300. def tableRelationCacheSize: Int
  301. def timestampType: AtomicType
  302. def toString(): String
    Definition Classes
    AnyRef → Any
  303. def topKSortFallbackThreshold: Int
  304. def truncateTableIgnorePermissionAcl: Boolean
  305. def uiExplainMode: String
  306. def unsetConf(entry: ConfigEntry[_]): Unit
  307. def unsetConf(key: String): Unit
  308. def useCompression: Boolean
  309. def useCurrentSQLConfigsForView: Boolean
  310. def useDeprecatedKafkaOffsetFetching: Boolean
  311. def useObjectHashAggregation: Boolean
  312. def useV1Command: Boolean
  313. def v2BucketingEnabled: Boolean
  314. def validatePartitionColumns: Boolean
  315. def variableSubstituteEnabled: Boolean
  316. def verifyPartitionPath: Boolean
  317. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  318. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  319. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  320. def warehousePath: String
  321. def wholeStageEnabled: Boolean
  322. def wholeStageMaxNumFields: Int
  323. def wholeStageSplitConsumeFuncByOperator: Boolean
  324. def wholeStageUseIdInClassName: Boolean
  325. def windowExecBufferInMemoryThreshold: Int
  326. def windowExecBufferSpillThreshold: Int
  327. def writeLegacyParquetFormat: Boolean

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] ) @Deprecated
    Deprecated

Inherited from Logging

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped