Packages

c

org.apache.spark.sql.hive

HiveContext

class HiveContext extends SQLContext with Logging

An instance of the Spark SQL execution engine that integrates with data stored in Hive. Configuration for Hive is read from hive-site.xml on the classpath.

Self Type
HiveContext
Annotations
@deprecated
Deprecated

(Since version 2.0.0) Use SparkSession.builder.enableHiveSupport instead

Linear Supertypes
SQLContext, Serializable, Serializable, Logging, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. HiveContext
  2. SQLContext
  3. Serializable
  4. Serializable
  5. Logging
  6. AnyRef
  7. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new HiveContext(sc: JavaSparkContext)
  2. new HiveContext(sc: SparkContext)

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  5. def baseRelationToDataFrame(baseRelation: BaseRelation): DataFrame
    Definition Classes
    SQLContext
  6. def cacheTable(tableName: String): Unit
    Definition Classes
    SQLContext
  7. def clearCache(): Unit
    Definition Classes
    SQLContext
  8. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native() @HotSpotIntrinsicCandidate()
  9. def createDataFrame(data: List[_], beanClass: Class[_]): DataFrame
    Definition Classes
    SQLContext
  10. def createDataFrame(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame
    Definition Classes
    SQLContext
  11. def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame
    Definition Classes
    SQLContext
  12. def createDataFrame(rows: List[Row], schema: StructType): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @DeveloperApi()
  13. def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @DeveloperApi()
  14. def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @DeveloperApi()
  15. def createDataFrame[A <: Product](data: Seq[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame
    Definition Classes
    SQLContext
  16. def createDataFrame[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame
    Definition Classes
    SQLContext
  17. def createDataset[T](data: List[T])(implicit arg0: Encoder[T]): Dataset[T]
    Definition Classes
    SQLContext
  18. def createDataset[T](data: RDD[T])(implicit arg0: Encoder[T]): Dataset[T]
    Definition Classes
    SQLContext
  19. def createDataset[T](data: Seq[T])(implicit arg0: Encoder[T]): Dataset[T]
    Definition Classes
    SQLContext
  20. def dropTempTable(tableName: String): Unit
    Definition Classes
    SQLContext
  21. def emptyDataFrame: DataFrame
    Definition Classes
    SQLContext
  22. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  23. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  24. def experimental: ExperimentalMethods
    Definition Classes
    SQLContext
    Annotations
    @Experimental() @transient() @Unstable()
  25. def getAllConfs: Map[String, String]
    Definition Classes
    SQLContext
  26. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  27. def getConf(key: String, defaultValue: String): String
    Definition Classes
    SQLContext
  28. def getConf(key: String): String
    Definition Classes
    SQLContext
  29. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  30. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  31. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  32. def isCached(tableName: String): Boolean
    Definition Classes
    SQLContext
  33. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  34. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  35. def listenerManager: ExecutionListenerManager
    Definition Classes
    SQLContext
  36. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  37. def logDebug(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  38. def logDebug(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  39. def logError(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  40. def logError(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  41. def logInfo(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  42. def logInfo(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  43. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  44. def logTrace(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  45. def logTrace(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  46. def logWarning(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  47. def logWarning(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  48. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  49. def newSession(): HiveContext

    Returns a new HiveContext as new session, which will have separated SQLConf, UDF/UDAF, temporary tables and SessionState, but sharing the same CacheManager, IsolatedClientLoader and Hive client (both of execution and metadata) with existing HiveContext.

    Returns a new HiveContext as new session, which will have separated SQLConf, UDF/UDAF, temporary tables and SessionState, but sharing the same CacheManager, IsolatedClientLoader and Hive client (both of execution and metadata) with existing HiveContext.

    Definition Classes
    HiveContext → SQLContext
  50. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  51. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  52. def range(start: Long, end: Long, step: Long, numPartitions: Int): DataFrame
    Definition Classes
    SQLContext
  53. def range(start: Long, end: Long, step: Long): DataFrame
    Definition Classes
    SQLContext
  54. def range(start: Long, end: Long): DataFrame
    Definition Classes
    SQLContext
  55. def range(end: Long): DataFrame
    Definition Classes
    SQLContext
  56. def read: DataFrameReader
    Definition Classes
    SQLContext
  57. def readStream: DataStreamReader
    Definition Classes
    SQLContext
  58. def refreshTable(tableName: String): Unit

    Invalidate and refresh all the cached the metadata of the given table.

    Invalidate and refresh all the cached the metadata of the given table. For performance reasons, Spark SQL or the external data source library it uses might cache certain metadata about a table, such as the location of blocks. When those change outside of Spark SQL, users should call this function to invalidate the cache.

    Since

    1.3.0

  59. def setConf(key: String, value: String): Unit
    Definition Classes
    SQLContext
  60. def setConf(props: Properties): Unit
    Definition Classes
    SQLContext
  61. def sparkContext: SparkContext
    Definition Classes
    SQLContext
  62. val sparkSession: SparkSession
    Definition Classes
    SQLContext
  63. def sql(sqlText: String): DataFrame
    Definition Classes
    SQLContext
  64. def streams: StreamingQueryManager
    Definition Classes
    SQLContext
  65. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  66. def table(tableName: String): DataFrame
    Definition Classes
    SQLContext
  67. def tableNames(databaseName: String): Array[String]
    Definition Classes
    SQLContext
  68. def tableNames(): Array[String]
    Definition Classes
    SQLContext
  69. def tables(databaseName: String): DataFrame
    Definition Classes
    SQLContext
  70. def tables(): DataFrame
    Definition Classes
    SQLContext
  71. def toString(): String
    Definition Classes
    AnyRef → Any
  72. def udf: UDFRegistration
    Definition Classes
    SQLContext
  73. def uncacheTable(tableName: String): Unit
    Definition Classes
    SQLContext
  74. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  75. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  76. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Deprecated Value Members

  1. def applySchema(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  2. def applySchema(rdd: RDD[_], beanClass: Class[_]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  3. def applySchema(rowRDD: JavaRDD[Row], schema: StructType): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  4. def applySchema(rowRDD: RDD[Row], schema: StructType): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  5. def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.2.0) use sparkSession.catalog.createTable instead.

  6. def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.2.0) use sparkSession.catalog.createTable instead.

  7. def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.2.0) use sparkSession.catalog.createTable instead.

  8. def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.2.0) use sparkSession.catalog.createTable instead.

  9. def createExternalTable(tableName: String, path: String, source: String): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.2.0) use sparkSession.catalog.createTable instead.

  10. def createExternalTable(tableName: String, path: String): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.2.0) use sparkSession.catalog.createTable instead.

  11. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] ) @Deprecated @deprecated
    Deprecated

    (Since version ) see corresponding Javadoc for more information.

  12. def jdbc(url: String, table: String, theParts: Array[String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.jdbc() instead.

  13. def jdbc(url: String, table: String, columnName: String, lowerBound: Long, upperBound: Long, numPartitions: Int): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.jdbc() instead.

  14. def jdbc(url: String, table: String): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.jdbc() instead.

  15. def jsonFile(path: String, samplingRatio: Double): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  16. def jsonFile(path: String, schema: StructType): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  17. def jsonFile(path: String): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  18. def jsonRDD(json: JavaRDD[String], samplingRatio: Double): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  19. def jsonRDD(json: RDD[String], samplingRatio: Double): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  20. def jsonRDD(json: JavaRDD[String], schema: StructType): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  21. def jsonRDD(json: RDD[String], schema: StructType): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  22. def jsonRDD(json: JavaRDD[String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  23. def jsonRDD(json: RDD[String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  24. def load(source: String, schema: StructType, options: Map[String, String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).schema(schema).options(options).load() instead.

  25. def load(source: String, schema: StructType, options: Map[String, String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).schema(schema).options(options).load() instead.

  26. def load(source: String, options: Map[String, String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).options(options).load() instead.

  27. def load(source: String, options: Map[String, String]): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).options(options).load() instead.

  28. def load(path: String, source: String): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).load(path) instead.

  29. def load(path: String): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.load(path) instead.

  30. def parquetFile(paths: String*): DataFrame
    Definition Classes
    SQLContext
    Annotations
    @deprecated @varargs()
    Deprecated

    (Since version 1.4.0) Use read.parquet() instead.

Inherited from SQLContext

Inherited from Serializable

Inherited from Serializable

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped