Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs;
019
020import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT;
021import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
022import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX;
023import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT;
024import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY;
025import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT;
026import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY;
027import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY;
028import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT;
029
030import java.io.IOException;
031import java.lang.reflect.Constructor;
032import java.lang.reflect.InvocationHandler;
033import java.lang.reflect.Proxy;
034import java.net.InetSocketAddress;
035import java.net.URI;
036import java.util.HashMap;
037import java.util.Map;
038import java.util.concurrent.TimeUnit;
039
040import org.apache.commons.logging.Log;
041import org.apache.commons.logging.LogFactory;
042import org.apache.hadoop.conf.Configuration;
043import org.apache.hadoop.hdfs.DFSClient.Conf;
044import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
045import org.apache.hadoop.hdfs.protocol.ClientProtocol;
046import org.apache.hadoop.hdfs.protocol.HdfsConstants;
047import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
048import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB;
049import org.apache.hadoop.hdfs.protocolPB.JournalProtocolPB;
050import org.apache.hadoop.hdfs.protocolPB.JournalProtocolTranslatorPB;
051import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB;
052import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
053import org.apache.hadoop.hdfs.server.namenode.NameNode;
054import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
055import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
056import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
057import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
058import org.apache.hadoop.io.Text;
059import org.apache.hadoop.io.retry.DefaultFailoverProxyProvider;
060import org.apache.hadoop.io.retry.FailoverProxyProvider;
061import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
062import org.apache.hadoop.io.retry.RetryPolicies;
063import org.apache.hadoop.io.retry.RetryPolicy;
064import org.apache.hadoop.io.retry.RetryProxy;
065import org.apache.hadoop.io.retry.RetryUtils;
066import org.apache.hadoop.ipc.ProtobufRpcEngine;
067import org.apache.hadoop.ipc.RPC;
068import org.apache.hadoop.ipc.RemoteException;
069import org.apache.hadoop.net.NetUtils;
070import org.apache.hadoop.security.RefreshUserMappingsProtocol;
071import org.apache.hadoop.security.SecurityUtil;
072import org.apache.hadoop.security.UserGroupInformation;
073import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
074import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolClientSideTranslatorPB;
075import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolPB;
076import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB;
077import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB;
078import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
079import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolPB;
080import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolClientSideTranslatorPB;
081import org.apache.hadoop.tools.GetUserMappingsProtocol;
082import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolClientSideTranslatorPB;
083import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolPB;
084
085import com.google.common.annotations.VisibleForTesting;
086import com.google.common.base.Preconditions;
087
088/**
089 * Create proxy objects to communicate with a remote NN. All remote access to an
090 * NN should be funneled through this class. Most of the time you'll want to use
091 * {@link NameNodeProxies#createProxy(Configuration, URI, Class)}, which will
092 * create either an HA- or non-HA-enabled client proxy as appropriate.
093 */
094public class NameNodeProxies {
095  
096  private static final Log LOG = LogFactory.getLog(NameNodeProxies.class);
097
098  /**
099   * Wrapper for a client proxy as well as its associated service ID.
100   * This is simply used as a tuple-like return type for
101   * {@link NameNodeProxies#createProxy} and
102   * {@link NameNodeProxies#createNonHAProxy}.
103   */
104  public static class ProxyAndInfo<PROXYTYPE> {
105    private final PROXYTYPE proxy;
106    private final Text dtService;
107    
108    public ProxyAndInfo(PROXYTYPE proxy, Text dtService) {
109      this.proxy = proxy;
110      this.dtService = dtService;
111    }
112    
113    public PROXYTYPE getProxy() {
114      return proxy;
115    }
116    
117    public Text getDelegationTokenService() {
118      return dtService;
119    }
120  }
121
122  /**
123   * Creates the namenode proxy with the passed protocol. This will handle
124   * creation of either HA- or non-HA-enabled proxy objects, depending upon
125   * if the provided URI is a configured logical URI.
126   * 
127   * @param conf the configuration containing the required IPC
128   *        properties, client failover configurations, etc.
129   * @param nameNodeUri the URI pointing either to a specific NameNode
130   *        or to a logical nameservice.
131   * @param xface the IPC interface which should be created
132   * @return an object containing both the proxy and the associated
133   *         delegation token service it corresponds to
134   * @throws IOException if there is an error creating the proxy
135   **/
136  @SuppressWarnings("unchecked")
137  public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
138      URI nameNodeUri, Class<T> xface) throws IOException {
139    Class<FailoverProxyProvider<T>> failoverProxyProviderClass =
140        getFailoverProxyProviderClass(conf, nameNodeUri, xface);
141  
142    if (failoverProxyProviderClass == null) {
143      // Non-HA case
144      return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
145          UserGroupInformation.getCurrentUser(), true);
146    } else {
147      // HA case
148      FailoverProxyProvider<T> failoverProxyProvider = NameNodeProxies
149          .createFailoverProxyProvider(conf, failoverProxyProviderClass, xface,
150              nameNodeUri);
151      Conf config = new Conf(conf);
152      T proxy = (T) RetryProxy.create(xface, failoverProxyProvider,
153          RetryPolicies.failoverOnNetworkException(
154              RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts,
155              config.maxRetryAttempts, config.failoverSleepBaseMillis,
156              config.failoverSleepMaxMillis));
157      
158      Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
159      return new ProxyAndInfo<T>(proxy, dtService);
160    }
161  }
162  
163  /**
164   * Generate a dummy namenode proxy instance that utilizes our hacked
165   * {@link LossyRetryInvocationHandler}. Proxy instance generated using this
166   * method will proactively drop RPC responses. Currently this method only
167   * support HA setup. null will be returned if the given configuration is not 
168   * for HA.
169   * 
170   * @param config the configuration containing the required IPC
171   *        properties, client failover configurations, etc.
172   * @param nameNodeUri the URI pointing either to a specific NameNode
173   *        or to a logical nameservice.
174   * @param xface the IPC interface which should be created
175   * @param numResponseToDrop The number of responses to drop for each RPC call
176   * @return an object containing both the proxy and the associated
177   *         delegation token service it corresponds to. Will return null of the
178   *         given configuration does not support HA.
179   * @throws IOException if there is an error creating the proxy
180   */
181  @SuppressWarnings("unchecked")
182  public static <T> ProxyAndInfo<T> createProxyWithLossyRetryHandler(
183      Configuration config, URI nameNodeUri, Class<T> xface,
184      int numResponseToDrop) throws IOException {
185    Preconditions.checkArgument(numResponseToDrop > 0);
186    Class<FailoverProxyProvider<T>> failoverProxyProviderClass = 
187        getFailoverProxyProviderClass(config, nameNodeUri, xface);
188    if (failoverProxyProviderClass != null) { // HA case
189      FailoverProxyProvider<T> failoverProxyProvider = 
190          createFailoverProxyProvider(config, failoverProxyProviderClass, 
191              xface, nameNodeUri);
192      int delay = config.getInt(
193          DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
194          DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
195      int maxCap = config.getInt(
196          DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
197          DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
198      int maxFailoverAttempts = config.getInt(
199          DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
200          DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
201      int maxRetryAttempts = config.getInt(
202          DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
203          DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
204      InvocationHandler dummyHandler = new LossyRetryInvocationHandler<T>(
205              numResponseToDrop, failoverProxyProvider,
206              RetryPolicies.failoverOnNetworkException(
207                  RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, 
208                  Math.max(numResponseToDrop + 1, maxRetryAttempts), delay, 
209                  maxCap));
210      
211      T proxy = (T) Proxy.newProxyInstance(
212          failoverProxyProvider.getInterface().getClassLoader(),
213          new Class[] { xface }, dummyHandler);
214      Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
215      return new ProxyAndInfo<T>(proxy, dtService);
216    } else {
217      LOG.warn("Currently creating proxy using " +
218                "LossyRetryInvocationHandler requires NN HA setup");
219      return null;
220    }
221  }
222
223  /**
224   * Creates an explicitly non-HA-enabled proxy object. Most of the time you
225   * don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
226   * 
227   * @param conf the configuration object
228   * @param nnAddr address of the remote NN to connect to
229   * @param xface the IPC interface which should be created
230   * @param ugi the user who is making the calls on the proxy object
231   * @param withRetries certain interfaces have a non-standard retry policy
232   * @return an object containing both the proxy and the associated
233   *         delegation token service it corresponds to
234   * @throws IOException
235   */
236  @SuppressWarnings("unchecked")
237  public static <T> ProxyAndInfo<T> createNonHAProxy(
238      Configuration conf, InetSocketAddress nnAddr, Class<T> xface,
239      UserGroupInformation ugi, boolean withRetries) throws IOException {
240    Text dtService = SecurityUtil.buildTokenService(nnAddr);
241  
242    T proxy;
243    if (xface == ClientProtocol.class) {
244      proxy = (T) createNNProxyWithClientProtocol(nnAddr, conf, ugi,
245          withRetries);
246    } else if (xface == JournalProtocol.class) {
247      proxy = (T) createNNProxyWithJournalProtocol(nnAddr, conf, ugi);
248    } else if (xface == NamenodeProtocol.class) {
249      proxy = (T) createNNProxyWithNamenodeProtocol(nnAddr, conf, ugi,
250          withRetries);
251    } else if (xface == GetUserMappingsProtocol.class) {
252      proxy = (T) createNNProxyWithGetUserMappingsProtocol(nnAddr, conf, ugi);
253    } else if (xface == RefreshUserMappingsProtocol.class) {
254      proxy = (T) createNNProxyWithRefreshUserMappingsProtocol(nnAddr, conf, ugi);
255    } else if (xface == RefreshAuthorizationPolicyProtocol.class) {
256      proxy = (T) createNNProxyWithRefreshAuthorizationPolicyProtocol(nnAddr,
257          conf, ugi);
258    } else if (xface == RefreshCallQueueProtocol.class) {
259      proxy = (T) createNNProxyWithRefreshCallQueueProtocol(nnAddr, conf, ugi);
260    } else {
261      String message = "Unsupported protocol found when creating the proxy " +
262          "connection to NameNode: " +
263          ((xface != null) ? xface.getClass().getName() : "null");
264      LOG.error(message);
265      throw new IllegalStateException(message);
266    }
267
268    return new ProxyAndInfo<T>(proxy, dtService);
269  }
270  
271  private static JournalProtocol createNNProxyWithJournalProtocol(
272      InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
273      throws IOException {
274    JournalProtocolPB proxy = (JournalProtocolPB) createNameNodeProxy(address,
275        conf, ugi, JournalProtocolPB.class);
276    return new JournalProtocolTranslatorPB(proxy);
277  }
278
279  private static RefreshAuthorizationPolicyProtocol
280      createNNProxyWithRefreshAuthorizationPolicyProtocol(InetSocketAddress address,
281          Configuration conf, UserGroupInformation ugi) throws IOException {
282    RefreshAuthorizationPolicyProtocolPB proxy = (RefreshAuthorizationPolicyProtocolPB)
283        createNameNodeProxy(address, conf, ugi, RefreshAuthorizationPolicyProtocolPB.class);
284    return new RefreshAuthorizationPolicyProtocolClientSideTranslatorPB(proxy);
285  }
286  
287  private static RefreshUserMappingsProtocol
288      createNNProxyWithRefreshUserMappingsProtocol(InetSocketAddress address,
289          Configuration conf, UserGroupInformation ugi) throws IOException {
290    RefreshUserMappingsProtocolPB proxy = (RefreshUserMappingsProtocolPB)
291        createNameNodeProxy(address, conf, ugi, RefreshUserMappingsProtocolPB.class);
292    return new RefreshUserMappingsProtocolClientSideTranslatorPB(proxy);
293  }
294
295  private static RefreshCallQueueProtocol
296      createNNProxyWithRefreshCallQueueProtocol(InetSocketAddress address,
297          Configuration conf, UserGroupInformation ugi) throws IOException {
298    RefreshCallQueueProtocolPB proxy = (RefreshCallQueueProtocolPB)
299        createNameNodeProxy(address, conf, ugi, RefreshCallQueueProtocolPB.class);
300    return new RefreshCallQueueProtocolClientSideTranslatorPB(proxy);
301  }
302
303  private static GetUserMappingsProtocol createNNProxyWithGetUserMappingsProtocol(
304      InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
305      throws IOException {
306    GetUserMappingsProtocolPB proxy = (GetUserMappingsProtocolPB)
307        createNameNodeProxy(address, conf, ugi, GetUserMappingsProtocolPB.class);
308    return new GetUserMappingsProtocolClientSideTranslatorPB(proxy);
309  }
310  
311  private static NamenodeProtocol createNNProxyWithNamenodeProtocol(
312      InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
313      boolean withRetries) throws IOException {
314    NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy(
315        address, conf, ugi, NamenodeProtocolPB.class);
316    if (withRetries) { // create the proxy with retries
317      RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200,
318          TimeUnit.MILLISECONDS);
319      Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap 
320                     = new HashMap<Class<? extends Exception>, RetryPolicy>();
321      RetryPolicy methodPolicy = RetryPolicies.retryByException(timeoutPolicy,
322          exceptionToPolicyMap);
323      Map<String, RetryPolicy> methodNameToPolicyMap 
324                     = new HashMap<String, RetryPolicy>();
325      methodNameToPolicyMap.put("getBlocks", methodPolicy);
326      methodNameToPolicyMap.put("getAccessKeys", methodPolicy);
327      proxy = (NamenodeProtocolPB) RetryProxy.create(NamenodeProtocolPB.class,
328          proxy, methodNameToPolicyMap);
329    }
330    return new NamenodeProtocolTranslatorPB(proxy);
331  }
332  
333  private static ClientProtocol createNNProxyWithClientProtocol(
334      InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
335      boolean withRetries) throws IOException {
336    RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);
337
338    final RetryPolicy defaultPolicy = 
339        RetryUtils.getDefaultRetryPolicy(
340            conf, 
341            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, 
342            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, 
343            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
344            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
345            SafeModeException.class);
346    
347    final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
348    ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
349        ClientNamenodeProtocolPB.class, version, address, ugi, conf,
350        NetUtils.getDefaultSocketFactory(conf),
351        org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy)
352            .getProxy();
353
354    if (withRetries) { // create the proxy with retries
355
356      RetryPolicy createPolicy = RetryPolicies
357          .retryUpToMaximumCountWithFixedSleep(5,
358              HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
359    
360      Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap 
361                 = new HashMap<Class<? extends Exception>, RetryPolicy>();
362      remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class,
363          createPolicy);
364    
365      Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap
366                 = new HashMap<Class<? extends Exception>, RetryPolicy>();
367      exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
368          .retryByRemoteException(defaultPolicy,
369              remoteExceptionToPolicyMap));
370      RetryPolicy methodPolicy = RetryPolicies.retryByException(
371          defaultPolicy, exceptionToPolicyMap);
372      Map<String, RetryPolicy> methodNameToPolicyMap 
373                 = new HashMap<String, RetryPolicy>();
374    
375      methodNameToPolicyMap.put("create", methodPolicy);
376    
377      proxy = (ClientNamenodeProtocolPB) RetryProxy.create(
378          ClientNamenodeProtocolPB.class,
379          new DefaultFailoverProxyProvider<ClientNamenodeProtocolPB>(
380              ClientNamenodeProtocolPB.class, proxy),
381          methodNameToPolicyMap,
382          defaultPolicy);
383    }
384    return new ClientNamenodeProtocolTranslatorPB(proxy);
385  }
386  
387  private static Object createNameNodeProxy(InetSocketAddress address,
388      Configuration conf, UserGroupInformation ugi, Class<?> xface)
389      throws IOException {
390    RPC.setProtocolEngine(conf, xface, ProtobufRpcEngine.class);
391    Object proxy = RPC.getProxy(xface, RPC.getProtocolVersion(xface), address,
392        ugi, conf, NetUtils.getDefaultSocketFactory(conf));
393    return proxy;
394  }
395
396  /** Gets the configured Failover proxy provider's class */
397  @VisibleForTesting
398  public static <T> Class<FailoverProxyProvider<T>> getFailoverProxyProviderClass(
399      Configuration conf, URI nameNodeUri, Class<T> xface) throws IOException {
400    if (nameNodeUri == null) {
401      return null;
402    }
403    String host = nameNodeUri.getHost();
404  
405    String configKey = DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "."
406        + host;
407    try {
408      @SuppressWarnings("unchecked")
409      Class<FailoverProxyProvider<T>> ret = (Class<FailoverProxyProvider<T>>) conf
410          .getClass(configKey, null, FailoverProxyProvider.class);
411      if (ret != null) {
412        // If we found a proxy provider, then this URI should be a logical NN.
413        // Given that, it shouldn't have a non-default port number.
414        int port = nameNodeUri.getPort();
415        if (port > 0 && port != NameNode.DEFAULT_PORT) {
416          throw new IOException("Port " + port + " specified in URI "
417              + nameNodeUri + " but host '" + host
418              + "' is a logical (HA) namenode"
419              + " and does not use port information.");
420        }
421      }
422      return ret;
423    } catch (RuntimeException e) {
424      if (e.getCause() instanceof ClassNotFoundException) {
425        throw new IOException("Could not load failover proxy provider class "
426            + conf.get(configKey) + " which is configured for authority "
427            + nameNodeUri, e);
428      } else {
429        throw e;
430      }
431    }
432  }
433
434  /** Creates the Failover proxy provider instance*/
435  @VisibleForTesting
436  public static <T> FailoverProxyProvider<T> createFailoverProxyProvider(
437      Configuration conf, Class<FailoverProxyProvider<T>> failoverProxyProviderClass,
438      Class<T> xface, URI nameNodeUri) throws IOException {
439    Preconditions.checkArgument(
440        xface.isAssignableFrom(NamenodeProtocols.class),
441        "Interface %s is not a NameNode protocol", xface);
442    try {
443      Constructor<FailoverProxyProvider<T>> ctor = failoverProxyProviderClass
444          .getConstructor(Configuration.class, URI.class, Class.class);
445      FailoverProxyProvider<T> provider = ctor.newInstance(conf, nameNodeUri,
446          xface);
447      return provider;
448    } catch (Exception e) {
449      String message = "Couldn't create proxy provider " + failoverProxyProviderClass;
450      if (LOG.isDebugEnabled()) {
451        LOG.debug(message, e);
452      }
453      if (e.getCause() instanceof IOException) {
454        throw (IOException) e.getCause();
455      } else {
456        throw new IOException(message, e);
457      }
458    }
459  }
460
461}