Package org.apache.polaris.spark

Class SparkCatalog

java.lang.Object
org.apache.polaris.spark.SparkCatalog
All Implemented Interfaces:
org.apache.iceberg.spark.SupportsReplaceView, org.apache.spark.sql.connector.catalog.CatalogPlugin, org.apache.spark.sql.connector.catalog.StagingTableCatalog, org.apache.spark.sql.connector.catalog.SupportsNamespaces, org.apache.spark.sql.connector.catalog.TableCatalog, org.apache.spark.sql.connector.catalog.ViewCatalog
public class SparkCatalog extends Object implements org.apache.spark.sql.connector.catalog.StagingTableCatalog, org.apache.spark.sql.connector.catalog.TableCatalog, org.apache.spark.sql.connector.catalog.SupportsNamespaces, org.apache.spark.sql.connector.catalog.ViewCatalog, org.apache.iceberg.spark.SupportsReplaceView
SparkCatalog Implementation that is able to interact with both Iceberg SparkCatalog and Polaris SparkCatalog. All namespaces and view related operations continue goes through the Iceberg SparkCatalog. For table operations, depends on the table format, the operation can be achieved with interaction with both Iceberg and Polaris SparkCatalog.

  • Field Summary

    Fields
    Modifier and Type
    Field
    Description
    protected String
    catalogName
     
    protected DeltaHelper
    deltaHelper
     
    protected org.apache.iceberg.spark.SparkCatalog
    icebergsSparkCatalog
     
    protected PolarisSparkCatalog
    polarisSparkCatalog
     

    Fields inherited from interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

    PROP_COMMENT, PROP_LOCATION, PROP_OWNER

    Fields inherited from interface org.apache.spark.sql.connector.catalog.TableCatalog

    OPTION_PREFIX, PROP_COMMENT, PROP_EXTERNAL, PROP_IS_MANAGED_LOCATION, PROP_LOCATION, PROP_OWNER, PROP_PROVIDER

    Fields inherited from interface org.apache.spark.sql.connector.catalog.ViewCatalog

    PROP_COMMENT, PROP_CREATE_ENGINE_VERSION, PROP_ENGINE_VERSION, PROP_OWNER, RESERVED_PROPERTIES

  • Constructor Summary

    Constructors
    Constructor
    Description
    SparkCatalog()
     

  • Method Summary

    Modifier and Type
    Method
    Description
    void
    alterNamespace(String[] namespace, org.apache.spark.sql.connector.catalog.NamespaceChange... changes)
     
    org.apache.spark.sql.connector.catalog.Table
    alterTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.TableChange... changes)
     
    org.apache.spark.sql.connector.catalog.View
    alterView(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.ViewChange... changes)
     
    void
    createNamespace(String[] namespace, Map<String,String> metadata)
     
    org.apache.spark.sql.connector.catalog.Table
    createTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties)
     
    org.apache.spark.sql.connector.catalog.View
    createView(org.apache.spark.sql.connector.catalog.Identifier ident, String sql, String currentCatalog, String[] currentNamespace, org.apache.spark.sql.types.StructType schema, String[] queryColumnNames, String[] columnAliases, String[] columnComments, Map<String,String> properties)
     
    String[]
    defaultNamespace()
     
    boolean
    dropNamespace(String[] namespace, boolean cascade)
     
    boolean
    dropTable(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    boolean
    dropView(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    void
    initialize(String name, org.apache.spark.sql.util.CaseInsensitiveStringMap options)
     
    void
    invalidateTable(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    String[][]
    listNamespaces()
     
    String[][]
    listNamespaces(String[] namespace)
     
    org.apache.spark.sql.connector.catalog.Identifier[]
    listTables(String[] namespace)
     
    org.apache.spark.sql.connector.catalog.Identifier[]
    listViews(String... namespace)
     
    Map<String,String>
    loadNamespaceMetadata(String[] namespace)
     
    org.apache.spark.sql.connector.catalog.Table
    loadTable(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    org.apache.spark.sql.connector.catalog.View
    loadView(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    String
    name()
     
    boolean
    purgeTable(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    void
    renameTable(org.apache.spark.sql.connector.catalog.Identifier from, org.apache.spark.sql.connector.catalog.Identifier to)
     
    void
    renameView(org.apache.spark.sql.connector.catalog.Identifier fromIdentifier, org.apache.spark.sql.connector.catalog.Identifier toIdentifier)
     
    org.apache.spark.sql.connector.catalog.View
    replaceView(org.apache.spark.sql.connector.catalog.Identifier ident, String sql, String currentCatalog, String[] currentNamespace, org.apache.spark.sql.types.StructType schema, String[] queryColumnNames, String[] columnAliases, String[] columnComments, Map<String,String> properties)
     
    org.apache.spark.sql.connector.catalog.StagedTable
    stageCreate(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties)
     
    org.apache.spark.sql.connector.catalog.StagedTable
    stageCreateOrReplace(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties)
     
    org.apache.spark.sql.connector.catalog.StagedTable
    stageReplace(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties)
     
    org.apache.spark.sql.util.CaseInsensitiveStringMap
    validateAndResolveCatalogOptions(org.apache.spark.sql.util.CaseInsensitiveStringMap options)
    Check whether invalid catalog configuration is provided, and return an option map with catalog type configured correctly.

    Methods inherited from class java.lang.Object

    clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

    Methods inherited from interface org.apache.spark.sql.connector.catalog.StagingTableCatalog

    stageCreate, stageCreateOrReplace, stageReplace

    Methods inherited from interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

    namespaceExists

    Methods inherited from interface org.apache.spark.sql.connector.catalog.TableCatalog

    capabilities, createTable, loadTable, loadTable, loadTable, tableExists, useNullableQuerySchema

    Methods inherited from interface org.apache.spark.sql.connector.catalog.ViewCatalog

    invalidateView, viewExists

  • Field Details

    • catalogName

      protected String catalogName

    • icebergsSparkCatalog

      protected org.apache.iceberg.spark.SparkCatalog icebergsSparkCatalog

    • polarisSparkCatalog

      protected PolarisSparkCatalog polarisSparkCatalog

    • deltaHelper

      protected DeltaHelper deltaHelper

  • Constructor Details

    • SparkCatalog

      public SparkCatalog()

  • Method Details

    • name

      public String name()
      Specified by:
      name in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

    • validateAndResolveCatalogOptions

      public org.apache.spark.sql.util.CaseInsensitiveStringMap validateAndResolveCatalogOptions(org.apache.spark.sql.util.CaseInsensitiveStringMap options)
      Check whether invalid catalog configuration is provided, and return an option map with catalog type configured correctly. This function mainly validates two parts: 1) No customized catalog implementation is provided. 2) No non-rest catalog type is configured.

    • initialize

      public void initialize(String name, org.apache.spark.sql.util.CaseInsensitiveStringMap options)
      Specified by:
      initialize in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

    • loadTable

      public org.apache.spark.sql.connector.catalog.Table loadTable(org.apache.spark.sql.connector.catalog.Identifier ident) throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException
      Specified by:
      loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchTableException

    • createTable

      public org.apache.spark.sql.connector.catalog.Table createTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties) throws org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException, org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      createTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • alterTable

      public org.apache.spark.sql.connector.catalog.Table alterTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.TableChange... changes) throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException
      Specified by:
      alterTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchTableException

    • dropTable

      public boolean dropTable(org.apache.spark.sql.connector.catalog.Identifier ident)
      Specified by:
      dropTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

    • renameTable

      public void renameTable(org.apache.spark.sql.connector.catalog.Identifier from, org.apache.spark.sql.connector.catalog.Identifier to) throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException, org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
      Specified by:
      renameTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchTableException
      org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

    • invalidateTable

      public void invalidateTable(org.apache.spark.sql.connector.catalog.Identifier ident)
      Specified by:
      invalidateTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

    • purgeTable

      public boolean purgeTable(org.apache.spark.sql.connector.catalog.Identifier ident)
      Specified by:
      purgeTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

    • listTables

      public org.apache.spark.sql.connector.catalog.Identifier[] listTables(String[] namespace)
      Specified by:
      listTables in interface org.apache.spark.sql.connector.catalog.TableCatalog

    • stageCreate

      public org.apache.spark.sql.connector.catalog.StagedTable stageCreate(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties) throws org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
      Specified by:
      stageCreate in interface org.apache.spark.sql.connector.catalog.StagingTableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

    • stageReplace

      public org.apache.spark.sql.connector.catalog.StagedTable stageReplace(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties) throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException
      Specified by:
      stageReplace in interface org.apache.spark.sql.connector.catalog.StagingTableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchTableException

    • stageCreateOrReplace

      public org.apache.spark.sql.connector.catalog.StagedTable stageCreateOrReplace(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties)
      Specified by:
      stageCreateOrReplace in interface org.apache.spark.sql.connector.catalog.StagingTableCatalog

    • defaultNamespace

      public String[] defaultNamespace()
      Specified by:
      defaultNamespace in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

    • listNamespaces

      public String[][] listNamespaces()
      Specified by:
      listNamespaces in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

    • listNamespaces

      public String[][] listNamespaces(String[] namespace) throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      listNamespaces in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • loadNamespaceMetadata

      public Map<String,String> loadNamespaceMetadata(String[] namespace) throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      loadNamespaceMetadata in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • createNamespace

      public void createNamespace(String[] namespace, Map<String,String> metadata) throws org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException
      Specified by:
      createNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
      Throws:
      org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException

    • alterNamespace

      public void alterNamespace(String[] namespace, org.apache.spark.sql.connector.catalog.NamespaceChange... changes) throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      alterNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • dropNamespace

      public boolean dropNamespace(String[] namespace, boolean cascade) throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      dropNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • listViews

      public org.apache.spark.sql.connector.catalog.Identifier[] listViews(String... namespace)
      Specified by:
      listViews in interface org.apache.spark.sql.connector.catalog.ViewCatalog

    • loadView

      public org.apache.spark.sql.connector.catalog.View loadView(org.apache.spark.sql.connector.catalog.Identifier ident) throws org.apache.spark.sql.catalyst.analysis.NoSuchViewException
      Specified by:
      loadView in interface org.apache.spark.sql.connector.catalog.ViewCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchViewException

    • createView

      public org.apache.spark.sql.connector.catalog.View createView(org.apache.spark.sql.connector.catalog.Identifier ident, String sql, String currentCatalog, String[] currentNamespace, org.apache.spark.sql.types.StructType schema, String[] queryColumnNames, String[] columnAliases, String[] columnComments, Map<String,String> properties) throws org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException, org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      createView in interface org.apache.spark.sql.connector.catalog.ViewCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • alterView

      public org.apache.spark.sql.connector.catalog.View alterView(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.ViewChange... changes) throws org.apache.spark.sql.catalyst.analysis.NoSuchViewException, IllegalArgumentException
      Specified by:
      alterView in interface org.apache.spark.sql.connector.catalog.ViewCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchViewException
      IllegalArgumentException

    • dropView

      public boolean dropView(org.apache.spark.sql.connector.catalog.Identifier ident)
      Specified by:
      dropView in interface org.apache.spark.sql.connector.catalog.ViewCatalog

    • renameView

      public void renameView(org.apache.spark.sql.connector.catalog.Identifier fromIdentifier, org.apache.spark.sql.connector.catalog.Identifier toIdentifier) throws org.apache.spark.sql.catalyst.analysis.NoSuchViewException, org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException
      Specified by:
      renameView in interface org.apache.spark.sql.connector.catalog.ViewCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchViewException
      org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException

    • replaceView

      public org.apache.spark.sql.connector.catalog.View replaceView(org.apache.spark.sql.connector.catalog.Identifier ident, String sql, String currentCatalog, String[] currentNamespace, org.apache.spark.sql.types.StructType schema, String[] queryColumnNames, String[] columnAliases, String[] columnComments, Map<String,String> properties) throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException, org.apache.spark.sql.catalyst.analysis.NoSuchViewException
      Specified by:
      replaceView in interface org.apache.iceberg.spark.SupportsReplaceView
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      org.apache.spark.sql.catalyst.analysis.NoSuchViewException