Package org.apache.polaris.spark

Class BasePolarisSparkCatalog

java.lang.Object
org.apache.polaris.spark.BasePolarisSparkCatalog
All Implemented Interfaces:
org.apache.spark.sql.connector.catalog.CatalogPlugin, org.apache.spark.sql.connector.catalog.StagingTableCatalog, org.apache.spark.sql.connector.catalog.SupportsNamespaces, org.apache.spark.sql.connector.catalog.TableCatalog
Direct Known Subclasses:
SparkCatalog
public abstract class BasePolarisSparkCatalog extends Object implements org.apache.spark.sql.connector.catalog.StagingTableCatalog, org.apache.spark.sql.connector.catalog.TableCatalog, org.apache.spark.sql.connector.catalog.SupportsNamespaces
SparkCatalog implementation that is able to interact with both Iceberg SparkCatalog and Polaris.

  • Field Summary

    Fields
    Modifier and Type
    Field
    Description
    protected String
    catalogName
     
    protected DeltaHelper
    deltaHelper
     
    protected HudiHelper
    hudiHelper
     
    protected org.apache.iceberg.spark.SparkCatalog
    icebergsSparkCatalog
     
    protected PaimonHelper
    paimonHelper
     
    protected PolarisSparkCatalog
    polarisSparkCatalog
     

    Fields inherited from interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

    PROP_COMMENT, PROP_LOCATION, PROP_OWNER

    Fields inherited from interface org.apache.spark.sql.connector.catalog.TableCatalog

    OPTION_PREFIX, PROP_COLLATION, PROP_COMMENT, PROP_EXTERNAL, PROP_IS_MANAGED_LOCATION, PROP_LOCATION, PROP_OWNER, PROP_PROVIDER

  • Constructor Summary

    Constructors
    Constructor
    Description
    BasePolarisSparkCatalog()
     

  • Method Summary

    Modifier and Type
    Method
    Description
    void
    alterNamespace(String[] namespace, org.apache.spark.sql.connector.catalog.NamespaceChange... changes)
     
    org.apache.spark.sql.connector.catalog.Table
    alterTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.TableChange... changes)
     
    org.apache.spark.sql.connector.catalog.View
    alterView(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.ViewChange... changes)
     
    protected abstract PolarisCatalogUtils
    createCatalogUtils()
     
    void
    createNamespace(String[] namespace, Map<String,String> metadata)
     
    org.apache.spark.sql.connector.catalog.Table
    createTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties)
     
    String[]
    defaultNamespace()
     
    boolean
    dropNamespace(String[] namespace, boolean cascade)
     
    boolean
    dropTable(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    boolean
    dropView(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    void
    initialize(String name, org.apache.spark.sql.util.CaseInsensitiveStringMap options)
     
    void
    invalidateTable(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    String[][]
    listNamespaces()
     
    String[][]
    listNamespaces(String[] namespace)
     
    org.apache.spark.sql.connector.catalog.Identifier[]
    listTables(String[] namespace)
     
    org.apache.spark.sql.connector.catalog.Identifier[]
    listViews(String... namespace)
     
    Map<String,String>
    loadNamespaceMetadata(String[] namespace)
     
    org.apache.spark.sql.connector.catalog.Table
    loadTable(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    org.apache.spark.sql.connector.catalog.View
    loadView(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    String
    name()
     
    boolean
    purgeTable(org.apache.spark.sql.connector.catalog.Identifier ident)
     
    void
    renameTable(org.apache.spark.sql.connector.catalog.Identifier from, org.apache.spark.sql.connector.catalog.Identifier to)
     
    void
    renameView(org.apache.spark.sql.connector.catalog.Identifier fromIdentifier, org.apache.spark.sql.connector.catalog.Identifier toIdentifier)
     
    org.apache.spark.sql.connector.catalog.View
    replaceView(org.apache.spark.sql.connector.catalog.Identifier ident, String sql, String currentCatalog, String[] currentNamespace, org.apache.spark.sql.types.StructType schema, String[] queryColumnNames, String[] columnAliases, String[] columnComments, Map<String,String> properties)
     
    org.apache.spark.sql.connector.catalog.StagedTable
    stageCreate(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties)
     
    org.apache.spark.sql.connector.catalog.StagedTable
    stageCreateOrReplace(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties)
     
    org.apache.spark.sql.connector.catalog.StagedTable
    stageReplace(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties)
     
    org.apache.spark.sql.util.CaseInsensitiveStringMap
    validateAndResolveCatalogOptions(org.apache.spark.sql.util.CaseInsensitiveStringMap options)
    Check whether invalid catalog configuration is provided, and return an option map with catalog type configured correctly.

    Methods inherited from class java.lang.Object

    clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

    Methods inherited from interface org.apache.spark.sql.connector.catalog.StagingTableCatalog

    stageCreate, stageCreateOrReplace, stageReplace, supportedCustomMetrics

    Methods inherited from interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

    namespaceExists

    Methods inherited from interface org.apache.spark.sql.connector.catalog.TableCatalog

    capabilities, createTable, loadTable, loadTable, loadTable, tableExists, useNullableQuerySchema

  • Field Details

    • catalogName

      protected String catalogName

    • icebergsSparkCatalog

      protected org.apache.iceberg.spark.SparkCatalog icebergsSparkCatalog

    • polarisSparkCatalog

      protected PolarisSparkCatalog polarisSparkCatalog

    • deltaHelper

      protected DeltaHelper deltaHelper

    • hudiHelper

      protected HudiHelper hudiHelper

    • paimonHelper

      protected PaimonHelper paimonHelper

  • Constructor Details

    • BasePolarisSparkCatalog

      public BasePolarisSparkCatalog()

  • Method Details

    • createCatalogUtils

      protected abstract PolarisCatalogUtils createCatalogUtils()

    • name

      public String name()
      Specified by:
      name in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

    • validateAndResolveCatalogOptions

      public org.apache.spark.sql.util.CaseInsensitiveStringMap validateAndResolveCatalogOptions(org.apache.spark.sql.util.CaseInsensitiveStringMap options)
      Check whether invalid catalog configuration is provided, and return an option map with catalog type configured correctly. This function mainly validates two parts: 1) No customized catalog implementation is provided. 2) No non-rest catalog type is configured.

    • initialize

      public void initialize(String name, org.apache.spark.sql.util.CaseInsensitiveStringMap options)
      Specified by:
      initialize in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

    • loadTable

      public org.apache.spark.sql.connector.catalog.Table loadTable(org.apache.spark.sql.connector.catalog.Identifier ident) throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException
      Specified by:
      loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchTableException

    • createTable

      public org.apache.spark.sql.connector.catalog.Table createTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties) throws org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException, org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      createTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • alterTable

      public org.apache.spark.sql.connector.catalog.Table alterTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.TableChange... changes) throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException
      Specified by:
      alterTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchTableException

    • dropTable

      public boolean dropTable(org.apache.spark.sql.connector.catalog.Identifier ident)
      Specified by:
      dropTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

    • renameTable

      public void renameTable(org.apache.spark.sql.connector.catalog.Identifier from, org.apache.spark.sql.connector.catalog.Identifier to) throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException, org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
      Specified by:
      renameTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchTableException
      org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

    • invalidateTable

      public void invalidateTable(org.apache.spark.sql.connector.catalog.Identifier ident)
      Specified by:
      invalidateTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

    • purgeTable

      public boolean purgeTable(org.apache.spark.sql.connector.catalog.Identifier ident)
      Specified by:
      purgeTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

    • listTables

      public org.apache.spark.sql.connector.catalog.Identifier[] listTables(String[] namespace)
      Specified by:
      listTables in interface org.apache.spark.sql.connector.catalog.TableCatalog

    • stageCreate

      public org.apache.spark.sql.connector.catalog.StagedTable stageCreate(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties) throws org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
      Specified by:
      stageCreate in interface org.apache.spark.sql.connector.catalog.StagingTableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

    • stageReplace

      public org.apache.spark.sql.connector.catalog.StagedTable stageReplace(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties) throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException
      Specified by:
      stageReplace in interface org.apache.spark.sql.connector.catalog.StagingTableCatalog
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchTableException

    • stageCreateOrReplace

      public org.apache.spark.sql.connector.catalog.StagedTable stageCreateOrReplace(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] transforms, Map<String,String> properties)
      Specified by:
      stageCreateOrReplace in interface org.apache.spark.sql.connector.catalog.StagingTableCatalog

    • defaultNamespace

      public String[] defaultNamespace()
      Specified by:
      defaultNamespace in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

    • listNamespaces

      public String[][] listNamespaces()
      Specified by:
      listNamespaces in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

    • listNamespaces

      public String[][] listNamespaces(String[] namespace) throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      listNamespaces in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • loadNamespaceMetadata

      public Map<String,String> loadNamespaceMetadata(String[] namespace) throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      loadNamespaceMetadata in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • createNamespace

      public void createNamespace(String[] namespace, Map<String,String> metadata) throws org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException
      Specified by:
      createNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
      Throws:
      org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException

    • alterNamespace

      public void alterNamespace(String[] namespace, org.apache.spark.sql.connector.catalog.NamespaceChange... changes) throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      alterNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • dropNamespace

      public boolean dropNamespace(String[] namespace, boolean cascade) throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      Specified by:
      dropNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

    • listViews

      public org.apache.spark.sql.connector.catalog.Identifier[] listViews(String... namespace)

    • loadView

      public org.apache.spark.sql.connector.catalog.View loadView(org.apache.spark.sql.connector.catalog.Identifier ident) throws org.apache.spark.sql.catalyst.analysis.NoSuchViewException
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchViewException

    • alterView

      public org.apache.spark.sql.connector.catalog.View alterView(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.ViewChange... changes) throws org.apache.spark.sql.catalyst.analysis.NoSuchViewException, IllegalArgumentException
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchViewException
      IllegalArgumentException

    • dropView

      public boolean dropView(org.apache.spark.sql.connector.catalog.Identifier ident)

    • renameView

      public void renameView(org.apache.spark.sql.connector.catalog.Identifier fromIdentifier, org.apache.spark.sql.connector.catalog.Identifier toIdentifier) throws org.apache.spark.sql.catalyst.analysis.NoSuchViewException, org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchViewException
      org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException

    • replaceView

      public org.apache.spark.sql.connector.catalog.View replaceView(org.apache.spark.sql.connector.catalog.Identifier ident, String sql, String currentCatalog, String[] currentNamespace, org.apache.spark.sql.types.StructType schema, String[] queryColumnNames, String[] columnAliases, String[] columnComments, Map<String,String> properties) throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException, org.apache.spark.sql.catalyst.analysis.NoSuchViewException
      Throws:
      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
      org.apache.spark.sql.catalyst.analysis.NoSuchViewException