Package org.apache.hadoop.fs.azurebfs.services

Class AbfsClient

java.lang.Object

org.apache.hadoop.fs.azurebfs.services.AbfsClient

All Implemented Interfaces:

java.io.Closeable, java.lang.AutoCloseable

public class AbfsClient
extends java.lang.Object
implements java.io.Closeable

AbfsClient.

Field Summary

Fields

Modifier and Type	Field	Description
static java.lang.String	HUNDRED_CONTINUE_USER_AGENT
static org.slf4j.Logger	LOG

Constructor Summary

Constructors

Constructor	Description
AbfsClient(java.net.URL baseUrl, SharedKeyCredentials sharedKeyCredentials, AbfsConfiguration abfsConfiguration, SASTokenProvider sasTokenProvider, EncryptionContextProvider encryptionContextProvider, AbfsClientContext abfsClientContext)
AbfsClient(java.net.URL baseUrl, SharedKeyCredentials sharedKeyCredentials, AbfsConfiguration abfsConfiguration, AccessTokenProvider tokenProvider, EncryptionContextProvider encryptionContextProvider, AbfsClientContext abfsClientContext)

Method Summary

Modifier and Type	Method	Description
AbfsRestOperation	acquireLease(java.lang.String path, int duration, TracingContext tracingContext)
<V> void	addCallback(org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture<V> future, org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback<V> callback)
AbfsRestOperation	append(java.lang.String path, byte[] buffer, AppendRequestParameters reqParams, java.lang.String cachedSasToken, ContextEncryptionAdapter contextEncryptionAdapter, TracingContext tracingContext)
boolean	appendSuccessCheckOp(AbfsRestOperation op, java.lang.String path, long length, TracingContext tracingContext)
AbfsRestOperation	breakLease(java.lang.String path, TracingContext tracingContext)
AbfsRestOperation	checkAccess(java.lang.String path, java.lang.String rwx, TracingContext tracingContext)	Talks to the server to check whether the permission specified in the rwx parameter is present for the path specified in the path parameter.
void	close()
java.lang.String	computeMD5Hash(byte[] data, int off, int len)	Compute MD5Hash of the given byte array starting from given offset up to given length.
protected java.util.List<AbfsHttpHeader>	createDefaultHeaders()	Create request headers for Rest Operation using the current API version.
AbfsRestOperation	createFilesystem(TracingContext tracingContext)
AbfsRestOperation	createPath(java.lang.String path, boolean isFile, boolean overwrite, AzureBlobFileSystemStore.Permissions permissions, boolean isAppendBlob, java.lang.String eTag, ContextEncryptionAdapter contextEncryptionAdapter, TracingContext tracingContext)	Method for calling createPath API to the backend.
protected java.net.URL	createRequestUrl(java.lang.String path, java.lang.String query)
AbfsRestOperation	deleteFilesystem(TracingContext tracingContext)
AbfsRestOperation	deleteIdempotencyCheckOp(AbfsRestOperation op)	Check if the delete request failure is post a retry and if delete failure qualifies to be a success response assuming idempotency.
AbfsRestOperation	deletePath(java.lang.String path, boolean recursive, java.lang.String continuation, TracingContext tracingContext, boolean isNamespaceEnabled)
AbfsRestOperation	flush(java.lang.String path, long position, boolean retainUncommittedData, boolean isClose, java.lang.String cachedSasToken, java.lang.String leaseId, ContextEncryptionAdapter contextEncryptionAdapter, TracingContext tracingContext)
protected AbfsConfiguration	getAbfsConfiguration()	Getter for abfsConfiguration from AbfsClient.
protected AbfsCounters	getAbfsCounters()	Getter for abfsCounters from AbfsClient.
protected AbfsPerfTracker	getAbfsPerfTracker()
java.lang.String	getAccessToken()
AbfsRestOperation	getAclStatus(java.lang.String path, boolean useUPN, TracingContext tracingContext)
AbfsRestOperation	getAclStatus(java.lang.String path, TracingContext tracingContext)
AuthType	getAuthType()
static java.lang.String	getDirectoryQueryParameter(java.lang.String path)	Get the directory query parameter used by the List Paths REST API and used as the path in the continuation token.
EncryptionContextProvider	getEncryptionContextProvider()
EncryptionType	getEncryptionType()
java.lang.String	getFileSystem()
AbfsRestOperation	getFilesystemProperties(TracingContext tracingContext)
protected java.lang.Boolean	getIsPaginatedDeleteEnabled()
int	getNumLeaseThreads()
AbfsRestOperation	getPathStatus(java.lang.String path, boolean includeProperties, TracingContext tracingContext, ContextEncryptionAdapter contextEncryptionAdapter)
AbfsRetryPolicy	getRetryPolicy(java.lang.String failureReason)	Returns the retry policy to be used for Abfs Rest Operation Failure.
SASTokenProvider	getSasTokenProvider()
protected AccessTokenProvider	getTokenProvider()
AbfsHttpConstants.ApiVersion	getxMsVersion()
AbfsRestOperation	listPath(java.lang.String relativePath, boolean recursive, int listMaxResults, java.lang.String continuation, TracingContext tracingContext)
AbfsRestOperation	read(java.lang.String path, long position, byte[] buffer, int bufferOffset, int bufferLength, java.lang.String eTag, java.lang.String cachedSasToken, ContextEncryptionAdapter contextEncryptionAdapter, TracingContext tracingContext)
AbfsRestOperation	releaseLease(java.lang.String path, java.lang.String leaseId, TracingContext tracingContext)
boolean	renameIdempotencyCheckOp(java.lang.String source, java.lang.String sourceEtag, AbfsRestOperation op, java.lang.String destination, TracingContext tracingContext)	Check if the rename request failure is post a retry and if earlier rename request might have succeeded at back-end.
AbfsClientRenameResult	renamePath(java.lang.String source, java.lang.String destination, java.lang.String continuation, TracingContext tracingContext, java.lang.String sourceEtag, boolean isMetadataIncompleteState, boolean isNamespaceEnabled)	Rename a file or directory.
AbfsRestOperation	renewLease(java.lang.String path, java.lang.String leaseId, TracingContext tracingContext)
<V> org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableScheduledFuture<V>	schedule(java.util.concurrent.Callable<V> callable, long delay, java.util.concurrent.TimeUnit timeUnit)
AbfsRestOperation	setAcl(java.lang.String path, java.lang.String aclSpecString, java.lang.String eTag, TracingContext tracingContext)
AbfsRestOperation	setAcl(java.lang.String path, java.lang.String aclSpecString, TracingContext tracingContext)
void	setEncryptionType(EncryptionType encryptionType)
AbfsRestOperation	setFilesystemProperties(java.lang.String properties, TracingContext tracingContext)
AbfsRestOperation	setOwner(java.lang.String path, java.lang.String owner, java.lang.String group, TracingContext tracingContext)
AbfsRestOperation	setPathProperties(java.lang.String path, java.lang.String properties, TracingContext tracingContext, ContextEncryptionAdapter contextEncryptionAdapter)
AbfsRestOperation	setPermission(java.lang.String path, java.lang.String permission, TracingContext tracingContext)
org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture<?>	submit(java.lang.Runnable runnable)
static java.lang.String	urlEncode(java.lang.String value)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

public static final org.slf4j.Logger LOG

HUNDRED_CONTINUE_USER_AGENT

public static final java.lang.String HUNDRED_CONTINUE_USER_AGENT

See Also:

Constant Field Values

Constructor Detail

AbfsClient

public AbfsClient(java.net.URL baseUrl,
                  SharedKeyCredentials sharedKeyCredentials,
                  AbfsConfiguration abfsConfiguration,
                  AccessTokenProvider tokenProvider,
                  EncryptionContextProvider encryptionContextProvider,
                  AbfsClientContext abfsClientContext)
           throws java.io.IOException

Throws:

java.io.IOException

AbfsClient

public AbfsClient(java.net.URL baseUrl,
                  SharedKeyCredentials sharedKeyCredentials,
                  AbfsConfiguration abfsConfiguration,
                  SASTokenProvider sasTokenProvider,
                  EncryptionContextProvider encryptionContextProvider,
                  AbfsClientContext abfsClientContext)
           throws java.io.IOException

Throws:

java.io.IOException

Method Detail

close

public void close()
           throws java.io.IOException

Specified by:

close in interface java.lang.AutoCloseable

Specified by:

close in interface java.io.Closeable

Throws:

java.io.IOException

getFileSystem

public java.lang.String getFileSystem()

getAbfsPerfTracker

protected AbfsPerfTracker getAbfsPerfTracker()

getRetryPolicy

public AbfsRetryPolicy getRetryPolicy(java.lang.String failureReason)

Returns the retry policy to be used for Abfs Rest Operation Failure.

Parameters:

failureReason - helps to decide which type of retryPolicy to be used.

Returns:

retry policy to be used.

setEncryptionType

public void setEncryptionType(EncryptionType encryptionType)

getEncryptionType

public EncryptionType getEncryptionType()

createDefaultHeaders

@VisibleForTesting
protected java.util.List<AbfsHttpHeader> createDefaultHeaders()

Create request headers for Rest Operation using the current API version.

Returns:

default request headers

createFilesystem

public AbfsRestOperation createFilesystem(TracingContext tracingContext)
                                   throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

setFilesystemProperties

public AbfsRestOperation setFilesystemProperties(java.lang.String properties,
                                                 TracingContext tracingContext)
                                          throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

listPath

public AbfsRestOperation listPath(java.lang.String relativePath,
                                  boolean recursive,
                                  int listMaxResults,
                                  java.lang.String continuation,
                                  TracingContext tracingContext)
                           throws java.io.IOException

Throws:

java.io.IOException

getFilesystemProperties

public AbfsRestOperation getFilesystemProperties(TracingContext tracingContext)
                                          throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

deleteFilesystem

public AbfsRestOperation deleteFilesystem(TracingContext tracingContext)
                                   throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

createPath

public AbfsRestOperation createPath(java.lang.String path,
                                    boolean isFile,
                                    boolean overwrite,
                                    AzureBlobFileSystemStore.Permissions permissions,
                                    boolean isAppendBlob,
                                    java.lang.String eTag,
                                    ContextEncryptionAdapter contextEncryptionAdapter,
                                    TracingContext tracingContext)
                             throws AzureBlobFileSystemException

Method for calling createPath API to the backend. Method can be called from:

1. create new file
2. overwrite file
3. create new directory

Parameters:

path - : path of the file / directory to be created / overwritten.

isFile - : defines if file or directory has to be created / overwritten.

overwrite - : defines if the file / directory to be overwritten.

permissions - : contains permission and umask

isAppendBlob - : defines if directory in the path is enabled for appendBlob

eTag - : required in case of overwrite of file / directory. Path would be overwritten only if the provided eTag is equal to the one present in backend for the path.

contextEncryptionAdapter - : object that contains the encryptionContext and encryptionKey created from the developer provided implementation of EncryptionContextProvider

tracingContext - : Object of TracingContext correlating to the current fs.create() request.

Returns:

object of AbfsRestOperation which contain all the information about the communication with the server. The information is in AbfsRestOperation.getResult()

Throws:

AzureBlobFileSystemException - throws back the exception it receives from the AbfsRestOperation.execute(TracingContext) method call.

acquireLease

public AbfsRestOperation acquireLease(java.lang.String path,
                                      int duration,
                                      TracingContext tracingContext)
                               throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

renewLease

public AbfsRestOperation renewLease(java.lang.String path,
                                    java.lang.String leaseId,
                                    TracingContext tracingContext)
                             throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

releaseLease

public AbfsRestOperation releaseLease(java.lang.String path,
                                      java.lang.String leaseId,
                                      TracingContext tracingContext)
                               throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

breakLease

public AbfsRestOperation breakLease(java.lang.String path,
                                    TracingContext tracingContext)
                             throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

renamePath

public AbfsClientRenameResult renamePath(java.lang.String source,
                                         java.lang.String destination,
                                         java.lang.String continuation,
                                         TracingContext tracingContext,
                                         java.lang.String sourceEtag,
                                         boolean isMetadataIncompleteState,
                                         boolean isNamespaceEnabled)
                                  throws java.io.IOException

Rename a file or directory. If a source etag is passed in, the operation will attempt to recover from a missing source file by probing the destination for existence and comparing etags. The second value in the result will be true to indicate that this took place. As rename recovery is only attempted if the source etag is non-empty, in normal rename operations rename recovery will never happen.

Parameters:

source - path to source file

destination - destination of rename.

continuation - continuation.

tracingContext - trace context

sourceEtag - etag of source file. may be null or empty

isMetadataIncompleteState - was there a rename failure due to incomplete metadata state?

isNamespaceEnabled - whether namespace enabled account or not

Returns:

AbfsClientRenameResult result of rename operation indicating the AbfsRest operation, rename recovery and incomplete metadata state failure.

Throws:

AzureBlobFileSystemException - failure, excluding any recovery from overload failures.

java.io.IOException

renameIdempotencyCheckOp

public boolean renameIdempotencyCheckOp(java.lang.String source,
                                        java.lang.String sourceEtag,
                                        AbfsRestOperation op,
                                        java.lang.String destination,
                                        TracingContext tracingContext)

Check if the rename request failure is post a retry and if earlier rename request might have succeeded at back-end. If a source etag was passed in, and the error was 404, get the etag of any file at the destination. If it matches the source etag, then the rename is considered a success. Exceptions raised in the probe of the destination are swallowed, so that they do not interfere with the original rename failures.

Parameters:

source - source path

op - Rename request REST operation response with non-null HTTP response

destination - rename destination path

sourceEtag - etag of source file. may be null or empty

tracingContext - Tracks identifiers for request header

Returns:

true if the file was successfully copied

append

public AbfsRestOperation append(java.lang.String path,
                                byte[] buffer,
                                AppendRequestParameters reqParams,
                                java.lang.String cachedSasToken,
                                ContextEncryptionAdapter contextEncryptionAdapter,
                                TracingContext tracingContext)
                         throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

appendSuccessCheckOp

public boolean appendSuccessCheckOp(AbfsRestOperation op,
                                    java.lang.String path,
                                    long length,
                                    TracingContext tracingContext)
                             throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

flush

public AbfsRestOperation flush(java.lang.String path,
                               long position,
                               boolean retainUncommittedData,
                               boolean isClose,
                               java.lang.String cachedSasToken,
                               java.lang.String leaseId,
                               ContextEncryptionAdapter contextEncryptionAdapter,
                               TracingContext tracingContext)
                        throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

setPathProperties

public AbfsRestOperation setPathProperties(java.lang.String path,
                                           java.lang.String properties,
                                           TracingContext tracingContext,
                                           ContextEncryptionAdapter contextEncryptionAdapter)
                                    throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

getPathStatus

public AbfsRestOperation getPathStatus(java.lang.String path,
                                       boolean includeProperties,
                                       TracingContext tracingContext,
                                       ContextEncryptionAdapter contextEncryptionAdapter)
                                throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

read

public AbfsRestOperation read(java.lang.String path,
                              long position,
                              byte[] buffer,
                              int bufferOffset,
                              int bufferLength,
                              java.lang.String eTag,
                              java.lang.String cachedSasToken,
                              ContextEncryptionAdapter contextEncryptionAdapter,
                              TracingContext tracingContext)
                       throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

deletePath

public AbfsRestOperation deletePath(java.lang.String path,
                                    boolean recursive,
                                    java.lang.String continuation,
                                    TracingContext tracingContext,
                                    boolean isNamespaceEnabled)
                             throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

deleteIdempotencyCheckOp

public AbfsRestOperation deleteIdempotencyCheckOp(AbfsRestOperation op)

Check if the delete request failure is post a retry and if delete failure qualifies to be a success response assuming idempotency. There are below scenarios where delete could be incorrectly deducted as success post request retry: 1. Target was originally not existing and initial delete request had to be re-tried. 2. Parallel delete issued from any other store interface rather than delete issued from this filesystem instance. These are few corner cases and usually returning a success at this stage should help the job to continue.

Parameters:

op - Delete request REST operation response with non-null HTTP response

Returns:

REST operation response post idempotency check

setOwner

public AbfsRestOperation setOwner(java.lang.String path,
                                  java.lang.String owner,
                                  java.lang.String group,
                                  TracingContext tracingContext)
                           throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

setPermission

public AbfsRestOperation setPermission(java.lang.String path,
                                       java.lang.String permission,
                                       TracingContext tracingContext)
                                throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

setAcl

public AbfsRestOperation setAcl(java.lang.String path,
                                java.lang.String aclSpecString,
                                TracingContext tracingContext)
                         throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

setAcl

public AbfsRestOperation setAcl(java.lang.String path,
                                java.lang.String aclSpecString,
                                java.lang.String eTag,
                                TracingContext tracingContext)
                         throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

getAclStatus

public AbfsRestOperation getAclStatus(java.lang.String path,
                                      TracingContext tracingContext)
                               throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

getAclStatus

public AbfsRestOperation getAclStatus(java.lang.String path,
                                      boolean useUPN,
                                      TracingContext tracingContext)
                               throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

checkAccess

public AbfsRestOperation checkAccess(java.lang.String path,
                                     java.lang.String rwx,
                                     TracingContext tracingContext)
                              throws AzureBlobFileSystemException

Talks to the server to check whether the permission specified in the rwx parameter is present for the path specified in the path parameter.

Parameters:

path - Path for which access check needs to be performed

rwx - The permission to be checked on the path

tracingContext - Tracks identifiers for request header

Returns:

The AbfsRestOperation object for the operation

Throws:

AzureBlobFileSystemException - in case of bad requests

getDirectoryQueryParameter

public static java.lang.String getDirectoryQueryParameter(java.lang.String path)

Get the directory query parameter used by the List Paths REST API and used as the path in the continuation token. If the input path is null or the root path "/", empty string is returned. If the input path begins with '/', the return value is the substring beginning at offset 1. Otherwise, the input path is returned.

Parameters:

path - the path to be listed.

Returns:

the value of the directory query parameter

createRequestUrl

@VisibleForTesting
protected java.net.URL createRequestUrl(java.lang.String path,
                                        java.lang.String query)
                                 throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

urlEncode

public static java.lang.String urlEncode(java.lang.String value)
                                  throws AzureBlobFileSystemException

Throws:

AzureBlobFileSystemException

getAccessToken

public java.lang.String getAccessToken()
                                throws java.io.IOException

Throws:

java.io.IOException

getIsPaginatedDeleteEnabled

protected java.lang.Boolean getIsPaginatedDeleteEnabled()

getAuthType

public AuthType getAuthType()

getEncryptionContextProvider

public EncryptionContextProvider getEncryptionContextProvider()

computeMD5Hash

@VisibleForTesting
public java.lang.String computeMD5Hash(byte[] data,
                                       int off,
                                       int len)
                                throws AbfsRestOperationException

Compute MD5Hash of the given byte array starting from given offset up to given length.

Parameters:

data - byte array from which data is fetched to compute MD5 Hash.

off - offset in the array from where actual data starts.

len - length of the data to be used to compute MD5Hash.

Returns:

MD5 Hash of the data as String.

Throws:

AbfsRestOperationException - if computation fails.

getSasTokenProvider

@VisibleForTesting
public SASTokenProvider getSasTokenProvider()

getAbfsCounters

protected AbfsCounters getAbfsCounters()

Getter for abfsCounters from AbfsClient.

Returns:

AbfsCounters instance.

getxMsVersion

public AbfsHttpConstants.ApiVersion getxMsVersion()

getAbfsConfiguration

protected AbfsConfiguration getAbfsConfiguration()

Getter for abfsConfiguration from AbfsClient.

Returns:

AbfsConfiguration instance

getNumLeaseThreads

public int getNumLeaseThreads()

schedule

public <V> org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableScheduledFuture<V> schedule(java.util.concurrent.Callable<V> callable,
                                                                                                                long delay,
                                                                                                                java.util.concurrent.TimeUnit timeUnit)

submit

public org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture<?> submit(java.lang.Runnable runnable)

addCallback

public <V> void addCallback(org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture<V> future,
                            org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback<V> callback)

getTokenProvider

@VisibleForTesting
protected AccessTokenProvider getTokenProvider()