001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs;
020
021 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
022 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
023 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY;
024 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT;
025 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
026 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
027 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
028 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
029 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY;
030 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
031 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
032 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
033
034 import java.io.IOException;
035 import java.io.PrintStream;
036 import java.io.UnsupportedEncodingException;
037 import java.net.InetSocketAddress;
038 import java.net.URI;
039 import java.net.URISyntaxException;
040 import java.security.SecureRandom;
041 import java.util.Collection;
042 import java.util.Collections;
043 import java.util.Comparator;
044 import java.util.HashMap;
045 import java.util.HashSet;
046 import java.util.List;
047 import java.util.Map;
048 import java.util.Random;
049 import java.util.Set;
050 import java.util.concurrent.TimeUnit;
051
052 import javax.net.SocketFactory;
053
054 import org.apache.commons.cli.CommandLine;
055 import org.apache.commons.cli.CommandLineParser;
056 import org.apache.commons.cli.Option;
057 import org.apache.commons.cli.Options;
058 import org.apache.commons.cli.ParseException;
059 import org.apache.commons.cli.PosixParser;
060 import org.apache.commons.logging.Log;
061 import org.apache.commons.logging.LogFactory;
062 import org.apache.hadoop.HadoopIllegalArgumentException;
063 import org.apache.hadoop.classification.InterfaceAudience;
064 import org.apache.hadoop.conf.Configuration;
065 import org.apache.hadoop.fs.BlockLocation;
066 import org.apache.hadoop.fs.FileSystem;
067 import org.apache.hadoop.fs.Path;
068 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
069 import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
070 import org.apache.hadoop.hdfs.protocol.DatanodeID;
071 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
072 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
073 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
074 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
075 import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB;
076 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
077 import org.apache.hadoop.hdfs.server.namenode.NameNode;
078 import org.apache.hadoop.io.retry.RetryPolicies;
079 import org.apache.hadoop.io.retry.RetryPolicy;
080 import org.apache.hadoop.io.retry.RetryProxy;
081 import org.apache.hadoop.ipc.ProtobufRpcEngine;
082 import org.apache.hadoop.ipc.RPC;
083 import org.apache.hadoop.net.NetUtils;
084 import org.apache.hadoop.net.NodeBase;
085 import org.apache.hadoop.security.SecurityUtil;
086 import org.apache.hadoop.security.UserGroupInformation;
087 import org.apache.hadoop.util.StringUtils;
088 import org.apache.hadoop.util.ToolRunner;
089
090 import com.google.common.base.Charsets;
091 import com.google.common.base.Joiner;
092 import com.google.common.base.Preconditions;
093 import com.google.common.collect.Lists;
094 import com.google.common.collect.Maps;
095 import com.google.common.primitives.SignedBytes;
096 import com.google.protobuf.BlockingService;
097
098 @InterfaceAudience.Private
099 public class DFSUtil {
100 public static final Log LOG = LogFactory.getLog(DFSUtil.class.getName());
101
102 public static final byte[] EMPTY_BYTES = {};
103
104 /** Compare two byte arrays by lexicographical order. */
105 public static int compareBytes(byte[] left, byte[] right) {
106 if (left == null) {
107 left = EMPTY_BYTES;
108 }
109 if (right == null) {
110 right = EMPTY_BYTES;
111 }
112 return SignedBytes.lexicographicalComparator().compare(left, right);
113 }
114
115 private DFSUtil() { /* Hidden constructor */ }
116 private static final ThreadLocal<Random> RANDOM = new ThreadLocal<Random>() {
117 @Override
118 protected Random initialValue() {
119 return new Random();
120 }
121 };
122
123 private static final ThreadLocal<SecureRandom> SECURE_RANDOM = new ThreadLocal<SecureRandom>() {
124 @Override
125 protected SecureRandom initialValue() {
126 return new SecureRandom();
127 }
128 };
129
130 /** @return a pseudo random number generator. */
131 public static Random getRandom() {
132 return RANDOM.get();
133 }
134
135 /** @return a pseudo secure random number generator. */
136 public static SecureRandom getSecureRandom() {
137 return SECURE_RANDOM.get();
138 }
139
140 /**
141 * Compartor for sorting DataNodeInfo[] based on decommissioned states.
142 * Decommissioned nodes are moved to the end of the array on sorting with
143 * this compartor.
144 */
145 public static final Comparator<DatanodeInfo> DECOM_COMPARATOR =
146 new Comparator<DatanodeInfo>() {
147 @Override
148 public int compare(DatanodeInfo a, DatanodeInfo b) {
149 return a.isDecommissioned() == b.isDecommissioned() ? 0 :
150 a.isDecommissioned() ? 1 : -1;
151 }
152 };
153
154
155 /**
156 * Comparator for sorting DataNodeInfo[] based on decommissioned/stale states.
157 * Decommissioned/stale nodes are moved to the end of the array on sorting
158 * with this comparator.
159 */
160 @InterfaceAudience.Private
161 public static class DecomStaleComparator implements Comparator<DatanodeInfo> {
162 private long staleInterval;
163
164 /**
165 * Constructor of DecomStaleComparator
166 *
167 * @param interval
168 * The time interval for marking datanodes as stale is passed from
169 * outside, since the interval may be changed dynamically
170 */
171 public DecomStaleComparator(long interval) {
172 this.staleInterval = interval;
173 }
174
175 @Override
176 public int compare(DatanodeInfo a, DatanodeInfo b) {
177 // Decommissioned nodes will still be moved to the end of the list
178 if (a.isDecommissioned()) {
179 return b.isDecommissioned() ? 0 : 1;
180 } else if (b.isDecommissioned()) {
181 return -1;
182 }
183 // Stale nodes will be moved behind the normal nodes
184 boolean aStale = a.isStale(staleInterval);
185 boolean bStale = b.isStale(staleInterval);
186 return aStale == bStale ? 0 : (aStale ? 1 : -1);
187 }
188 }
189
190 /**
191 * Address matcher for matching an address to local address
192 */
193 static final AddressMatcher LOCAL_ADDRESS_MATCHER = new AddressMatcher() {
194 @Override
195 public boolean match(InetSocketAddress s) {
196 return NetUtils.isLocalAddress(s.getAddress());
197 };
198 };
199
200 /**
201 * Whether the pathname is valid. Currently prohibits relative paths,
202 * names which contain a ":" or "//", or other non-canonical paths.
203 */
204 public static boolean isValidName(String src) {
205 // Path must be absolute.
206 if (!src.startsWith(Path.SEPARATOR)) {
207 return false;
208 }
209
210 // Check for ".." "." ":" "/"
211 String[] components = StringUtils.split(src, '/');
212 for (int i = 0; i < components.length; i++) {
213 String element = components[i];
214 if (element.equals(".") ||
215 (element.indexOf(":") >= 0) ||
216 (element.indexOf("/") >= 0)) {
217 return false;
218 }
219 // ".." is allowed in path starting with /.reserved/.inodes
220 if (element.equals("..")) {
221 if (components.length > 4
222 && components[1].equals(FSDirectory.DOT_RESERVED_STRING)
223 && components[2].equals(FSDirectory.DOT_INODES_STRING)) {
224 continue;
225 }
226 return false;
227 }
228 // The string may start or end with a /, but not have
229 // "//" in the middle.
230 if (element.isEmpty() && i != components.length - 1 &&
231 i != 0) {
232 return false;
233 }
234 }
235 return true;
236 }
237
238 /**
239 * Converts a byte array to a string using UTF8 encoding.
240 */
241 public static String bytes2String(byte[] bytes) {
242 return bytes2String(bytes, 0, bytes.length);
243 }
244
245 /**
246 * Decode a specific range of bytes of the given byte array to a string
247 * using UTF8.
248 *
249 * @param bytes The bytes to be decoded into characters
250 * @param offset The index of the first byte to decode
251 * @param length The number of bytes to decode
252 * @return The decoded string
253 */
254 public static String bytes2String(byte[] bytes, int offset, int length) {
255 try {
256 return new String(bytes, offset, length, "UTF8");
257 } catch(UnsupportedEncodingException e) {
258 assert false : "UTF8 encoding is not supported ";
259 }
260 return null;
261 }
262
263 /**
264 * Converts a string to a byte array using UTF8 encoding.
265 */
266 public static byte[] string2Bytes(String str) {
267 return str.getBytes(Charsets.UTF_8);
268 }
269
270 /**
271 * Given a list of path components returns a path as a UTF8 String
272 */
273 public static String byteArray2PathString(byte[][] pathComponents) {
274 if (pathComponents.length == 0) {
275 return "";
276 } else if (pathComponents.length == 1
277 && (pathComponents[0] == null || pathComponents[0].length == 0)) {
278 return Path.SEPARATOR;
279 }
280 StringBuilder result = new StringBuilder();
281 for (int i = 0; i < pathComponents.length; i++) {
282 result.append(new String(pathComponents[i], Charsets.UTF_8));
283 if (i < pathComponents.length - 1) {
284 result.append(Path.SEPARATOR_CHAR);
285 }
286 }
287 return result.toString();
288 }
289
290 /**
291 * Given a list of path components returns a byte array
292 */
293 public static byte[] byteArray2bytes(byte[][] pathComponents) {
294 if (pathComponents.length == 0) {
295 return EMPTY_BYTES;
296 } else if (pathComponents.length == 1
297 && (pathComponents[0] == null || pathComponents[0].length == 0)) {
298 return new byte[]{(byte) Path.SEPARATOR_CHAR};
299 }
300 int length = 0;
301 for (int i = 0; i < pathComponents.length; i++) {
302 length += pathComponents[i].length;
303 if (i < pathComponents.length - 1) {
304 length++; // for SEPARATOR
305 }
306 }
307 byte[] path = new byte[length];
308 int index = 0;
309 for (int i = 0; i < pathComponents.length; i++) {
310 System.arraycopy(pathComponents[i], 0, path, index,
311 pathComponents[i].length);
312 index += pathComponents[i].length;
313 if (i < pathComponents.length - 1) {
314 path[index] = (byte) Path.SEPARATOR_CHAR;
315 index++;
316 }
317 }
318 return path;
319 }
320
321 /** Convert an object representing a path to a string. */
322 public static String path2String(final Object path) {
323 return path == null? null
324 : path instanceof String? (String)path
325 : path instanceof byte[][]? byteArray2PathString((byte[][])path)
326 : path.toString();
327 }
328
329 /**
330 * Splits the array of bytes into array of arrays of bytes
331 * on byte separator
332 * @param bytes the array of bytes to split
333 * @param separator the delimiting byte
334 */
335 public static byte[][] bytes2byteArray(byte[] bytes, byte separator) {
336 return bytes2byteArray(bytes, bytes.length, separator);
337 }
338
339 /**
340 * Splits first len bytes in bytes to array of arrays of bytes
341 * on byte separator
342 * @param bytes the byte array to split
343 * @param len the number of bytes to split
344 * @param separator the delimiting byte
345 */
346 public static byte[][] bytes2byteArray(byte[] bytes,
347 int len,
348 byte separator) {
349 assert len <= bytes.length;
350 int splits = 0;
351 if (len == 0) {
352 return new byte[][]{null};
353 }
354 // Count the splits. Omit multiple separators and the last one
355 for (int i = 0; i < len; i++) {
356 if (bytes[i] == separator) {
357 splits++;
358 }
359 }
360 int last = len - 1;
361 while (last > -1 && bytes[last--] == separator) {
362 splits--;
363 }
364 if (splits == 0 && bytes[0] == separator) {
365 return new byte[][]{null};
366 }
367 splits++;
368 byte[][] result = new byte[splits][];
369 int startIndex = 0;
370 int nextIndex = 0;
371 int index = 0;
372 // Build the splits
373 while (index < splits) {
374 while (nextIndex < len && bytes[nextIndex] != separator) {
375 nextIndex++;
376 }
377 result[index] = new byte[nextIndex - startIndex];
378 System.arraycopy(bytes, startIndex, result[index], 0, nextIndex
379 - startIndex);
380 index++;
381 startIndex = nextIndex + 1;
382 nextIndex = startIndex;
383 }
384 return result;
385 }
386
387 /**
388 * Convert a LocatedBlocks to BlockLocations[]
389 * @param blocks a LocatedBlocks
390 * @return an array of BlockLocations
391 */
392 public static BlockLocation[] locatedBlocks2Locations(LocatedBlocks blocks) {
393 if (blocks == null) {
394 return new BlockLocation[0];
395 }
396 return locatedBlocks2Locations(blocks.getLocatedBlocks());
397 }
398
399 /**
400 * Convert a List<LocatedBlock> to BlockLocation[]
401 * @param blocks A List<LocatedBlock> to be converted
402 * @return converted array of BlockLocation
403 */
404 public static BlockLocation[] locatedBlocks2Locations(List<LocatedBlock> blocks) {
405 if (blocks == null) {
406 return new BlockLocation[0];
407 }
408 int nrBlocks = blocks.size();
409 BlockLocation[] blkLocations = new BlockLocation[nrBlocks];
410 if (nrBlocks == 0) {
411 return blkLocations;
412 }
413 int idx = 0;
414 for (LocatedBlock blk : blocks) {
415 assert idx < nrBlocks : "Incorrect index";
416 DatanodeInfo[] locations = blk.getLocations();
417 String[] hosts = new String[locations.length];
418 String[] xferAddrs = new String[locations.length];
419 String[] racks = new String[locations.length];
420 for (int hCnt = 0; hCnt < locations.length; hCnt++) {
421 hosts[hCnt] = locations[hCnt].getHostName();
422 xferAddrs[hCnt] = locations[hCnt].getXferAddr();
423 NodeBase node = new NodeBase(xferAddrs[hCnt],
424 locations[hCnt].getNetworkLocation());
425 racks[hCnt] = node.toString();
426 }
427 blkLocations[idx] = new BlockLocation(xferAddrs, hosts, racks,
428 blk.getStartOffset(),
429 blk.getBlockSize(),
430 blk.isCorrupt());
431 idx++;
432 }
433 return blkLocations;
434 }
435
436 /**
437 * Returns collection of nameservice Ids from the configuration.
438 * @param conf configuration
439 * @return collection of nameservice Ids, or null if not specified
440 */
441 public static Collection<String> getNameServiceIds(Configuration conf) {
442 return conf.getTrimmedStringCollection(DFS_NAMESERVICES);
443 }
444
445 /**
446 * @return <code>coll</code> if it is non-null and non-empty. Otherwise,
447 * returns a list with a single null value.
448 */
449 private static Collection<String> emptyAsSingletonNull(Collection<String> coll) {
450 if (coll == null || coll.isEmpty()) {
451 return Collections.singletonList(null);
452 } else {
453 return coll;
454 }
455 }
456
457 /**
458 * Namenode HighAvailability related configuration.
459 * Returns collection of namenode Ids from the configuration. One logical id
460 * for each namenode in the in the HA setup.
461 *
462 * @param conf configuration
463 * @param nsId the nameservice ID to look at, or null for non-federated
464 * @return collection of namenode Ids
465 */
466 public static Collection<String> getNameNodeIds(Configuration conf, String nsId) {
467 String key = addSuffix(DFS_HA_NAMENODES_KEY_PREFIX, nsId);
468 return conf.getTrimmedStringCollection(key);
469 }
470
471 /**
472 * Given a list of keys in the order of preference, returns a value
473 * for the key in the given order from the configuration.
474 * @param defaultValue default value to return, when key was not found
475 * @param keySuffix suffix to add to the key, if it is not null
476 * @param conf Configuration
477 * @param keys list of keys in the order of preference
478 * @return value of the key or default if a key was not found in configuration
479 */
480 private static String getConfValue(String defaultValue, String keySuffix,
481 Configuration conf, String... keys) {
482 String value = null;
483 for (String key : keys) {
484 key = addSuffix(key, keySuffix);
485 value = conf.get(key);
486 if (value != null) {
487 break;
488 }
489 }
490 if (value == null) {
491 value = defaultValue;
492 }
493 return value;
494 }
495
496 /** Add non empty and non null suffix to a key */
497 private static String addSuffix(String key, String suffix) {
498 if (suffix == null || suffix.isEmpty()) {
499 return key;
500 }
501 assert !suffix.startsWith(".") :
502 "suffix '" + suffix + "' should not already have '.' prepended.";
503 return key + "." + suffix;
504 }
505
506 /** Concatenate list of suffix strings '.' separated */
507 private static String concatSuffixes(String... suffixes) {
508 if (suffixes == null) {
509 return null;
510 }
511 return Joiner.on(".").skipNulls().join(suffixes);
512 }
513
514 /**
515 * Return configuration key of format key.suffix1.suffix2...suffixN
516 */
517 public static String addKeySuffixes(String key, String... suffixes) {
518 String keySuffix = concatSuffixes(suffixes);
519 return addSuffix(key, keySuffix);
520 }
521
522 /**
523 * Returns the configured address for all NameNodes in the cluster.
524 * @param conf configuration
525 * @param defaultAddress default address to return in case key is not found.
526 * @param keys Set of keys to look for in the order of preference
527 * @return a map(nameserviceId to map(namenodeId to InetSocketAddress))
528 */
529 private static Map<String, Map<String, InetSocketAddress>>
530 getAddresses(Configuration conf,
531 String defaultAddress, String... keys) {
532 Collection<String> nameserviceIds = getNameServiceIds(conf);
533
534 // Look for configurations of the form <key>[.<nameserviceId>][.<namenodeId>]
535 // across all of the configured nameservices and namenodes.
536 Map<String, Map<String, InetSocketAddress>> ret = Maps.newLinkedHashMap();
537 for (String nsId : emptyAsSingletonNull(nameserviceIds)) {
538 Map<String, InetSocketAddress> isas =
539 getAddressesForNameserviceId(conf, nsId, defaultAddress, keys);
540 if (!isas.isEmpty()) {
541 ret.put(nsId, isas);
542 }
543 }
544 return ret;
545 }
546
547 private static Map<String, InetSocketAddress> getAddressesForNameserviceId(
548 Configuration conf, String nsId, String defaultValue,
549 String[] keys) {
550 Collection<String> nnIds = getNameNodeIds(conf, nsId);
551 Map<String, InetSocketAddress> ret = Maps.newHashMap();
552 for (String nnId : emptyAsSingletonNull(nnIds)) {
553 String suffix = concatSuffixes(nsId, nnId);
554 String address = getConfValue(defaultValue, suffix, conf, keys);
555 if (address != null) {
556 InetSocketAddress isa = NetUtils.createSocketAddr(address);
557 if (isa.isUnresolved()) {
558 LOG.warn("Namenode for " + nsId +
559 " remains unresolved for ID " + nnId +
560 ". Check your hdfs-site.xml file to " +
561 "ensure namenodes are configured properly.");
562 }
563 ret.put(nnId, isa);
564 }
565 }
566 return ret;
567 }
568
569 /**
570 * @return a collection of all configured NN Kerberos principals.
571 */
572 public static Set<String> getAllNnPrincipals(Configuration conf) throws IOException {
573 Set<String> principals = new HashSet<String>();
574 for (String nsId : DFSUtil.getNameServiceIds(conf)) {
575 if (HAUtil.isHAEnabled(conf, nsId)) {
576 for (String nnId : DFSUtil.getNameNodeIds(conf, nsId)) {
577 Configuration confForNn = new Configuration(conf);
578 NameNode.initializeGenericKeys(confForNn, nsId, nnId);
579 String principal = SecurityUtil.getServerPrincipal(confForNn
580 .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY),
581 NameNode.getAddress(confForNn).getHostName());
582 principals.add(principal);
583 }
584 } else {
585 Configuration confForNn = new Configuration(conf);
586 NameNode.initializeGenericKeys(confForNn, nsId, null);
587 String principal = SecurityUtil.getServerPrincipal(confForNn
588 .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY),
589 NameNode.getAddress(confForNn).getHostName());
590 principals.add(principal);
591 }
592 }
593
594 return principals;
595 }
596
597 /**
598 * Returns list of InetSocketAddress corresponding to HA NN RPC addresses from
599 * the configuration.
600 *
601 * @param conf configuration
602 * @return list of InetSocketAddresses
603 */
604 public static Map<String, Map<String, InetSocketAddress>> getHaNnRpcAddresses(
605 Configuration conf) {
606 return getAddresses(conf, null, DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
607 }
608
609 /**
610 * Returns list of InetSocketAddress corresponding to backup node rpc
611 * addresses from the configuration.
612 *
613 * @param conf configuration
614 * @return list of InetSocketAddresses
615 * @throws IOException on error
616 */
617 public static Map<String, Map<String, InetSocketAddress>> getBackupNodeAddresses(
618 Configuration conf) throws IOException {
619 Map<String, Map<String, InetSocketAddress>> addressList = getAddresses(conf,
620 null, DFS_NAMENODE_BACKUP_ADDRESS_KEY);
621 if (addressList.isEmpty()) {
622 throw new IOException("Incorrect configuration: backup node address "
623 + DFS_NAMENODE_BACKUP_ADDRESS_KEY + " is not configured.");
624 }
625 return addressList;
626 }
627
628 /**
629 * Returns list of InetSocketAddresses of corresponding to secondary namenode
630 * http addresses from the configuration.
631 *
632 * @param conf configuration
633 * @return list of InetSocketAddresses
634 * @throws IOException on error
635 */
636 public static Map<String, Map<String, InetSocketAddress>> getSecondaryNameNodeAddresses(
637 Configuration conf) throws IOException {
638 Map<String, Map<String, InetSocketAddress>> addressList = getAddresses(conf, null,
639 DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY);
640 if (addressList.isEmpty()) {
641 throw new IOException("Incorrect configuration: secondary namenode address "
642 + DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY + " is not configured.");
643 }
644 return addressList;
645 }
646
647 /**
648 * Returns list of InetSocketAddresses corresponding to namenodes from the
649 * configuration. Note this is to be used by datanodes to get the list of
650 * namenode addresses to talk to.
651 *
652 * Returns namenode address specifically configured for datanodes (using
653 * service ports), if found. If not, regular RPC address configured for other
654 * clients is returned.
655 *
656 * @param conf configuration
657 * @return list of InetSocketAddress
658 * @throws IOException on error
659 */
660 public static Map<String, Map<String, InetSocketAddress>> getNNServiceRpcAddresses(
661 Configuration conf) throws IOException {
662 // Use default address as fall back
663 String defaultAddress;
664 try {
665 defaultAddress = NetUtils.getHostPortString(NameNode.getAddress(conf));
666 } catch (IllegalArgumentException e) {
667 defaultAddress = null;
668 }
669
670 Map<String, Map<String, InetSocketAddress>> addressList =
671 getAddresses(conf, defaultAddress,
672 DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);
673 if (addressList.isEmpty()) {
674 throw new IOException("Incorrect configuration: namenode address "
675 + DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + " or "
676 + DFS_NAMENODE_RPC_ADDRESS_KEY
677 + " is not configured.");
678 }
679 return addressList;
680 }
681
682 /**
683 * Flatten the given map, as returned by other functions in this class,
684 * into a flat list of {@link ConfiguredNNAddress} instances.
685 */
686 public static List<ConfiguredNNAddress> flattenAddressMap(
687 Map<String, Map<String, InetSocketAddress>> map) {
688 List<ConfiguredNNAddress> ret = Lists.newArrayList();
689
690 for (Map.Entry<String, Map<String, InetSocketAddress>> entry :
691 map.entrySet()) {
692 String nsId = entry.getKey();
693 Map<String, InetSocketAddress> nnMap = entry.getValue();
694 for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
695 String nnId = e2.getKey();
696 InetSocketAddress addr = e2.getValue();
697
698 ret.add(new ConfiguredNNAddress(nsId, nnId, addr));
699 }
700 }
701 return ret;
702 }
703
704 /**
705 * Format the given map, as returned by other functions in this class,
706 * into a string suitable for debugging display. The format of this string
707 * should not be considered an interface, and is liable to change.
708 */
709 public static String addressMapToString(
710 Map<String, Map<String, InetSocketAddress>> map) {
711 StringBuilder b = new StringBuilder();
712 for (Map.Entry<String, Map<String, InetSocketAddress>> entry :
713 map.entrySet()) {
714 String nsId = entry.getKey();
715 Map<String, InetSocketAddress> nnMap = entry.getValue();
716 b.append("Nameservice <").append(nsId).append(">:").append("\n");
717 for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
718 b.append(" NN ID ").append(e2.getKey())
719 .append(" => ").append(e2.getValue()).append("\n");
720 }
721 }
722 return b.toString();
723 }
724
725 public static String nnAddressesAsString(Configuration conf) {
726 Map<String, Map<String, InetSocketAddress>> addresses =
727 getHaNnRpcAddresses(conf);
728 return addressMapToString(addresses);
729 }
730
731 /**
732 * Represent one of the NameNodes configured in the cluster.
733 */
734 public static class ConfiguredNNAddress {
735 private final String nameserviceId;
736 private final String namenodeId;
737 private final InetSocketAddress addr;
738
739 private ConfiguredNNAddress(String nameserviceId, String namenodeId,
740 InetSocketAddress addr) {
741 this.nameserviceId = nameserviceId;
742 this.namenodeId = namenodeId;
743 this.addr = addr;
744 }
745
746 public String getNameserviceId() {
747 return nameserviceId;
748 }
749
750 public String getNamenodeId() {
751 return namenodeId;
752 }
753
754 public InetSocketAddress getAddress() {
755 return addr;
756 }
757
758 @Override
759 public String toString() {
760 return "ConfiguredNNAddress[nsId=" + nameserviceId + ";" +
761 "nnId=" + namenodeId + ";addr=" + addr + "]";
762 }
763 }
764
765 /**
766 * Get a URI for each configured nameservice. If a nameservice is
767 * HA-enabled, then the logical URI of the nameservice is returned. If the
768 * nameservice is not HA-enabled, then a URI corresponding to an RPC address
769 * of the single NN for that nameservice is returned, preferring the service
770 * RPC address over the client RPC address.
771 *
772 * @param conf configuration
773 * @return a collection of all configured NN URIs, preferring service
774 * addresses
775 */
776 public static Collection<URI> getNsServiceRpcUris(Configuration conf) {
777 return getNameServiceUris(conf,
778 DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
779 DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
780 }
781
782 /**
783 * Get a URI for each configured nameservice. If a nameservice is
784 * HA-enabled, then the logical URI of the nameservice is returned. If the
785 * nameservice is not HA-enabled, then a URI corresponding to the address of
786 * the single NN for that nameservice is returned.
787 *
788 * @param conf configuration
789 * @param keys configuration keys to try in order to get the URI for non-HA
790 * nameservices
791 * @return a collection of all configured NN URIs
792 */
793 public static Collection<URI> getNameServiceUris(Configuration conf,
794 String... keys) {
795 Set<URI> ret = new HashSet<URI>();
796
797 // We're passed multiple possible configuration keys for any given NN or HA
798 // nameservice, and search the config in order of these keys. In order to
799 // make sure that a later config lookup (e.g. fs.defaultFS) doesn't add a
800 // URI for a config key for which we've already found a preferred entry, we
801 // keep track of non-preferred keys here.
802 Set<URI> nonPreferredUris = new HashSet<URI>();
803
804 for (String nsId : getNameServiceIds(conf)) {
805 if (HAUtil.isHAEnabled(conf, nsId)) {
806 // Add the logical URI of the nameservice.
807 try {
808 ret.add(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId));
809 } catch (URISyntaxException ue) {
810 throw new IllegalArgumentException(ue);
811 }
812 } else {
813 // Add the URI corresponding to the address of the NN.
814 boolean uriFound = false;
815 for (String key : keys) {
816 String addr = conf.get(concatSuffixes(key, nsId));
817 if (addr != null) {
818 URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME,
819 NetUtils.createSocketAddr(addr));
820 if (!uriFound) {
821 uriFound = true;
822 ret.add(uri);
823 } else {
824 nonPreferredUris.add(uri);
825 }
826 }
827 }
828 }
829 }
830
831 // Add the generic configuration keys.
832 boolean uriFound = false;
833 for (String key : keys) {
834 String addr = conf.get(key);
835 if (addr != null) {
836 URI uri = createUri("hdfs", NetUtils.createSocketAddr(addr));
837 if (!uriFound) {
838 uriFound = true;
839 ret.add(uri);
840 } else {
841 nonPreferredUris.add(uri);
842 }
843 }
844 }
845
846 // Add the default URI if it is an HDFS URI.
847 URI defaultUri = FileSystem.getDefaultUri(conf);
848 // checks if defaultUri is ip:port format
849 // and convert it to hostname:port format
850 if (defaultUri != null && (defaultUri.getPort() != -1)) {
851 defaultUri = createUri(defaultUri.getScheme(),
852 NetUtils.createSocketAddr(defaultUri.getHost(),
853 defaultUri.getPort()));
854 }
855 if (defaultUri != null &&
856 HdfsConstants.HDFS_URI_SCHEME.equals(defaultUri.getScheme()) &&
857 !nonPreferredUris.contains(defaultUri)) {
858 ret.add(defaultUri);
859 }
860
861 return ret;
862 }
863
864 /**
865 * Given the InetSocketAddress this method returns the nameservice Id
866 * corresponding to the key with matching address, by doing a reverse
867 * lookup on the list of nameservices until it finds a match.
868 *
869 * Since the process of resolving URIs to Addresses is slightly expensive,
870 * this utility method should not be used in performance-critical routines.
871 *
872 * @param conf - configuration
873 * @param address - InetSocketAddress for configured communication with NN.
874 * Configured addresses are typically given as URIs, but we may have to
875 * compare against a URI typed in by a human, or the server name may be
876 * aliased, so we compare unambiguous InetSocketAddresses instead of just
877 * comparing URI substrings.
878 * @param keys - list of configured communication parameters that should
879 * be checked for matches. For example, to compare against RPC addresses,
880 * provide the list DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
881 * DFS_NAMENODE_RPC_ADDRESS_KEY. Use the generic parameter keys,
882 * not the NameServiceId-suffixed keys.
883 * @return nameserviceId, or null if no match found
884 */
885 public static String getNameServiceIdFromAddress(final Configuration conf,
886 final InetSocketAddress address, String... keys) {
887 // Configuration with a single namenode and no nameserviceId
888 String[] ids = getSuffixIDs(conf, address, keys);
889 return (ids != null) ? ids[0] : null;
890 }
891
892 /**
893 * return server http or https address from the configuration for a
894 * given namenode rpc address.
895 * @param conf
896 * @param namenodeAddr - namenode RPC address
897 * @param httpsAddress -If true, and if security is enabled, returns server
898 * https address. If false, returns server http address.
899 * @return server http or https address
900 * @throws IOException
901 */
902 public static String getInfoServer(InetSocketAddress namenodeAddr,
903 Configuration conf, boolean httpsAddress) throws IOException {
904 boolean securityOn = UserGroupInformation.isSecurityEnabled();
905 String httpAddressKey = (securityOn && httpsAddress) ?
906 DFS_NAMENODE_HTTPS_ADDRESS_KEY : DFS_NAMENODE_HTTP_ADDRESS_KEY;
907 String httpAddressDefault = (securityOn && httpsAddress) ?
908 DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT : DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
909
910 String suffixes[];
911 if (namenodeAddr != null) {
912 // if non-default namenode, try reverse look up
913 // the nameServiceID if it is available
914 suffixes = getSuffixIDs(conf, namenodeAddr,
915 DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
916 DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
917 } else {
918 suffixes = new String[2];
919 }
920 String configuredInfoAddr = getSuffixedConf(conf, httpAddressKey,
921 httpAddressDefault, suffixes);
922 if (namenodeAddr != null) {
923 return substituteForWildcardAddress(configuredInfoAddr,
924 namenodeAddr.getHostName());
925 } else {
926 return configuredInfoAddr;
927 }
928 }
929
930
931 /**
932 * Substitute a default host in the case that an address has been configured
933 * with a wildcard. This is used, for example, when determining the HTTP
934 * address of the NN -- if it's configured to bind to 0.0.0.0, we want to
935 * substitute the hostname from the filesystem URI rather than trying to
936 * connect to 0.0.0.0.
937 * @param configuredAddress the address found in the configuration
938 * @param defaultHost the host to substitute with, if configuredAddress
939 * is a local/wildcard address.
940 * @return the substituted address
941 * @throws IOException if it is a wildcard address and security is enabled
942 */
943 public static String substituteForWildcardAddress(String configuredAddress,
944 String defaultHost) throws IOException {
945 InetSocketAddress sockAddr = NetUtils.createSocketAddr(configuredAddress);
946 InetSocketAddress defaultSockAddr = NetUtils.createSocketAddr(defaultHost
947 + ":0");
948 if (sockAddr.getAddress().isAnyLocalAddress()) {
949 if (UserGroupInformation.isSecurityEnabled() &&
950 defaultSockAddr.getAddress().isAnyLocalAddress()) {
951 throw new IOException("Cannot use a wildcard address with security. " +
952 "Must explicitly set bind address for Kerberos");
953 }
954 return defaultHost + ":" + sockAddr.getPort();
955 } else {
956 return configuredAddress;
957 }
958 }
959
960 private static String getSuffixedConf(Configuration conf,
961 String key, String defaultVal, String[] suffixes) {
962 String ret = conf.get(DFSUtil.addKeySuffixes(key, suffixes));
963 if (ret != null) {
964 return ret;
965 }
966 return conf.get(key, defaultVal);
967 }
968
969 /**
970 * Sets the node specific setting into generic configuration key. Looks up
971 * value of "key.nameserviceId.namenodeId" and if found sets that value into
972 * generic key in the conf. If this is not found, falls back to
973 * "key.nameserviceId" and then the unmodified key.
974 *
975 * Note that this only modifies the runtime conf.
976 *
977 * @param conf
978 * Configuration object to lookup specific key and to set the value
979 * to the key passed. Note the conf object is modified.
980 * @param nameserviceId
981 * nameservice Id to construct the node specific key. Pass null if
982 * federation is not configuration.
983 * @param nnId
984 * namenode Id to construct the node specific key. Pass null if
985 * HA is not configured.
986 * @param keys
987 * The key for which node specific value is looked up
988 */
989 public static void setGenericConf(Configuration conf,
990 String nameserviceId, String nnId, String... keys) {
991 for (String key : keys) {
992 String value = conf.get(addKeySuffixes(key, nameserviceId, nnId));
993 if (value != null) {
994 conf.set(key, value);
995 continue;
996 }
997 value = conf.get(addKeySuffixes(key, nameserviceId));
998 if (value != null) {
999 conf.set(key, value);
1000 }
1001 }
1002 }
1003
1004 /** Return used as percentage of capacity */
1005 public static float getPercentUsed(long used, long capacity) {
1006 return capacity <= 0 ? 100 : (used * 100.0f)/capacity;
1007 }
1008
1009 /** Return remaining as percentage of capacity */
1010 public static float getPercentRemaining(long remaining, long capacity) {
1011 return capacity <= 0 ? 0 : (remaining * 100.0f)/capacity;
1012 }
1013
1014 /** Convert percentage to a string. */
1015 public static String percent2String(double percentage) {
1016 return StringUtils.format("%.2f%%", percentage);
1017 }
1018
1019 /**
1020 * Round bytes to GiB (gibibyte)
1021 * @param bytes number of bytes
1022 * @return number of GiB
1023 */
1024 public static int roundBytesToGB(long bytes) {
1025 return Math.round((float)bytes/ 1024 / 1024 / 1024);
1026 }
1027
1028 /** Create a {@link ClientDatanodeProtocol} proxy */
1029 public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
1030 DatanodeID datanodeid, Configuration conf, int socketTimeout,
1031 boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException {
1032 return new ClientDatanodeProtocolTranslatorPB(datanodeid, conf, socketTimeout,
1033 connectToDnViaHostname, locatedBlock);
1034 }
1035
1036 /** Create {@link ClientDatanodeProtocol} proxy using kerberos ticket */
1037 static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
1038 DatanodeID datanodeid, Configuration conf, int socketTimeout,
1039 boolean connectToDnViaHostname) throws IOException {
1040 return new ClientDatanodeProtocolTranslatorPB(
1041 datanodeid, conf, socketTimeout, connectToDnViaHostname);
1042 }
1043
1044 /** Create a {@link ClientDatanodeProtocol} proxy */
1045 public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
1046 InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
1047 SocketFactory factory) throws IOException {
1048 return new ClientDatanodeProtocolTranslatorPB(addr, ticket, conf, factory);
1049 }
1050
1051 /**
1052 * Get nameservice Id for the {@link NameNode} based on namenode RPC address
1053 * matching the local node address.
1054 */
1055 public static String getNamenodeNameServiceId(Configuration conf) {
1056 return getNameServiceId(conf, DFS_NAMENODE_RPC_ADDRESS_KEY);
1057 }
1058
1059 /**
1060 * Get nameservice Id for the BackupNode based on backup node RPC address
1061 * matching the local node address.
1062 */
1063 public static String getBackupNameServiceId(Configuration conf) {
1064 return getNameServiceId(conf, DFS_NAMENODE_BACKUP_ADDRESS_KEY);
1065 }
1066
1067 /**
1068 * Get nameservice Id for the secondary node based on secondary http address
1069 * matching the local node address.
1070 */
1071 public static String getSecondaryNameServiceId(Configuration conf) {
1072 return getNameServiceId(conf, DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY);
1073 }
1074
1075 /**
1076 * Get the nameservice Id by matching the {@code addressKey} with the
1077 * the address of the local node.
1078 *
1079 * If {@link DFSConfigKeys#DFS_NAMESERVICE_ID} is not specifically
1080 * configured, and more than one nameservice Id is configured, this method
1081 * determines the nameservice Id by matching the local node's address with the
1082 * configured addresses. When a match is found, it returns the nameservice Id
1083 * from the corresponding configuration key.
1084 *
1085 * @param conf Configuration
1086 * @param addressKey configuration key to get the address.
1087 * @return nameservice Id on success, null if federation is not configured.
1088 * @throws HadoopIllegalArgumentException on error
1089 */
1090 private static String getNameServiceId(Configuration conf, String addressKey) {
1091 String nameserviceId = conf.get(DFS_NAMESERVICE_ID);
1092 if (nameserviceId != null) {
1093 return nameserviceId;
1094 }
1095 Collection<String> nsIds = getNameServiceIds(conf);
1096 if (1 == nsIds.size()) {
1097 return nsIds.toArray(new String[1])[0];
1098 }
1099 String nnId = conf.get(DFS_HA_NAMENODE_ID_KEY);
1100
1101 return getSuffixIDs(conf, addressKey, null, nnId, LOCAL_ADDRESS_MATCHER)[0];
1102 }
1103
1104 /**
1105 * Returns nameservice Id and namenode Id when the local host matches the
1106 * configuration parameter {@code addressKey}.<nameservice Id>.<namenode Id>
1107 *
1108 * @param conf Configuration
1109 * @param addressKey configuration key corresponding to the address.
1110 * @param knownNsId only look at configs for the given nameservice, if not-null
1111 * @param knownNNId only look at configs for the given namenode, if not null
1112 * @param matcher matching criteria for matching the address
1113 * @return Array with nameservice Id and namenode Id on success. First element
1114 * in the array is nameservice Id and second element is namenode Id.
1115 * Null value indicates that the configuration does not have the the
1116 * Id.
1117 * @throws HadoopIllegalArgumentException on error
1118 */
1119 static String[] getSuffixIDs(final Configuration conf, final String addressKey,
1120 String knownNsId, String knownNNId,
1121 final AddressMatcher matcher) {
1122 String nameserviceId = null;
1123 String namenodeId = null;
1124 int found = 0;
1125
1126 Collection<String> nsIds = getNameServiceIds(conf);
1127 for (String nsId : emptyAsSingletonNull(nsIds)) {
1128 if (knownNsId != null && !knownNsId.equals(nsId)) {
1129 continue;
1130 }
1131
1132 Collection<String> nnIds = getNameNodeIds(conf, nsId);
1133 for (String nnId : emptyAsSingletonNull(nnIds)) {
1134 if (LOG.isTraceEnabled()) {
1135 LOG.trace(String.format("addressKey: %s nsId: %s nnId: %s",
1136 addressKey, nsId, nnId));
1137 }
1138 if (knownNNId != null && !knownNNId.equals(nnId)) {
1139 continue;
1140 }
1141 String key = addKeySuffixes(addressKey, nsId, nnId);
1142 String addr = conf.get(key);
1143 if (addr == null) {
1144 continue;
1145 }
1146 InetSocketAddress s = null;
1147 try {
1148 s = NetUtils.createSocketAddr(addr);
1149 } catch (Exception e) {
1150 LOG.warn("Exception in creating socket address " + addr, e);
1151 continue;
1152 }
1153 if (!s.isUnresolved() && matcher.match(s)) {
1154 nameserviceId = nsId;
1155 namenodeId = nnId;
1156 found++;
1157 }
1158 }
1159 }
1160 if (found > 1) { // Only one address must match the local address
1161 String msg = "Configuration has multiple addresses that match "
1162 + "local node's address. Please configure the system with "
1163 + DFS_NAMESERVICE_ID + " and "
1164 + DFS_HA_NAMENODE_ID_KEY;
1165 throw new HadoopIllegalArgumentException(msg);
1166 }
1167 return new String[] { nameserviceId, namenodeId };
1168 }
1169
1170 /**
1171 * For given set of {@code keys} adds nameservice Id and or namenode Id
1172 * and returns {nameserviceId, namenodeId} when address match is found.
1173 * @see #getSuffixIDs(Configuration, String, AddressMatcher)
1174 */
1175 static String[] getSuffixIDs(final Configuration conf,
1176 final InetSocketAddress address, final String... keys) {
1177 AddressMatcher matcher = new AddressMatcher() {
1178 @Override
1179 public boolean match(InetSocketAddress s) {
1180 return address.equals(s);
1181 }
1182 };
1183
1184 for (String key : keys) {
1185 String[] ids = getSuffixIDs(conf, key, null, null, matcher);
1186 if (ids != null && (ids [0] != null || ids[1] != null)) {
1187 return ids;
1188 }
1189 }
1190 return null;
1191 }
1192
1193 private interface AddressMatcher {
1194 public boolean match(InetSocketAddress s);
1195 }
1196
1197 /** Create a URI from the scheme and address */
1198 public static URI createUri(String scheme, InetSocketAddress address) {
1199 try {
1200 return new URI(scheme, null, address.getHostName(), address.getPort(),
1201 null, null, null);
1202 } catch (URISyntaxException ue) {
1203 throw new IllegalArgumentException(ue);
1204 }
1205 }
1206
1207 /**
1208 * Add protobuf based protocol to the {@link org.apache.hadoop.ipc.RPC.Server}
1209 * @param conf configuration
1210 * @param protocol Protocol interface
1211 * @param service service that implements the protocol
1212 * @param server RPC server to which the protocol & implementation is added to
1213 * @throws IOException
1214 */
1215 public static void addPBProtocol(Configuration conf, Class<?> protocol,
1216 BlockingService service, RPC.Server server) throws IOException {
1217 RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine.class);
1218 server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocol, service);
1219 }
1220
1221 /**
1222 * Map a logical namenode ID to its service address. Use the given
1223 * nameservice if specified, or the configured one if none is given.
1224 *
1225 * @param conf Configuration
1226 * @param nsId which nameservice nnId is a part of, optional
1227 * @param nnId the namenode ID to get the service addr for
1228 * @return the service addr, null if it could not be determined
1229 */
1230 public static String getNamenodeServiceAddr(final Configuration conf,
1231 String nsId, String nnId) {
1232
1233 if (nsId == null) {
1234 nsId = getOnlyNameServiceIdOrNull(conf);
1235 }
1236
1237 String serviceAddrKey = concatSuffixes(
1238 DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nsId, nnId);
1239
1240 String addrKey = concatSuffixes(
1241 DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, nnId);
1242
1243 String serviceRpcAddr = conf.get(serviceAddrKey);
1244 if (serviceRpcAddr == null) {
1245 serviceRpcAddr = conf.get(addrKey);
1246 }
1247 return serviceRpcAddr;
1248 }
1249
1250 /**
1251 * If the configuration refers to only a single nameservice, return the
1252 * name of that nameservice. If it refers to 0 or more than 1, return null.
1253 */
1254 public static String getOnlyNameServiceIdOrNull(Configuration conf) {
1255 Collection<String> nsIds = getNameServiceIds(conf);
1256 if (1 == nsIds.size()) {
1257 return nsIds.toArray(new String[1])[0];
1258 } else {
1259 // No nameservice ID was given and more than one is configured
1260 return null;
1261 }
1262 }
1263
1264 public static Options helpOptions = new Options();
1265 public static Option helpOpt = new Option("h", "help", false,
1266 "get help information");
1267
1268 static {
1269 helpOptions.addOption(helpOpt);
1270 }
1271
1272 /**
1273 * Parse the arguments for commands
1274 *
1275 * @param args the argument to be parsed
1276 * @param helpDescription help information to be printed out
1277 * @param out Printer
1278 * @param printGenericCommandUsage whether to print the
1279 * generic command usage defined in ToolRunner
1280 * @return true when the argument matches help option, false if not
1281 */
1282 public static boolean parseHelpArgument(String[] args,
1283 String helpDescription, PrintStream out, boolean printGenericCommandUsage) {
1284 if (args.length == 1) {
1285 try {
1286 CommandLineParser parser = new PosixParser();
1287 CommandLine cmdLine = parser.parse(helpOptions, args);
1288 if (cmdLine.hasOption(helpOpt.getOpt())
1289 || cmdLine.hasOption(helpOpt.getLongOpt())) {
1290 // should print out the help information
1291 out.println(helpDescription + "\n");
1292 if (printGenericCommandUsage) {
1293 ToolRunner.printGenericCommandUsage(out);
1294 }
1295 return true;
1296 }
1297 } catch (ParseException pe) {
1298 return false;
1299 }
1300 }
1301 return false;
1302 }
1303
1304 /**
1305 * Get DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION from configuration.
1306 *
1307 * @param conf Configuration
1308 * @return Value of DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION
1309 */
1310 public static float getInvalidateWorkPctPerIteration(Configuration conf) {
1311 float blocksInvalidateWorkPct = conf.getFloat(
1312 DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,
1313 DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT);
1314 Preconditions.checkArgument(
1315 (blocksInvalidateWorkPct > 0 && blocksInvalidateWorkPct <= 1.0f),
1316 DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION +
1317 " = '" + blocksInvalidateWorkPct + "' is invalid. " +
1318 "It should be a positive, non-zero float value, not greater than 1.0f, " +
1319 "to indicate a percentage.");
1320 return blocksInvalidateWorkPct;
1321 }
1322
1323 /**
1324 * Get DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION from
1325 * configuration.
1326 *
1327 * @param conf Configuration
1328 * @return Value of DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION
1329 */
1330 public static int getReplWorkMultiplier(Configuration conf) {
1331 int blocksReplWorkMultiplier = conf.getInt(
1332 DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
1333 DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT);
1334 Preconditions.checkArgument(
1335 (blocksReplWorkMultiplier > 0),
1336 DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION +
1337 " = '" + blocksReplWorkMultiplier + "' is invalid. " +
1338 "It should be a positive, non-zero integer value.");
1339 return blocksReplWorkMultiplier;
1340 }
1341
1342 /**
1343 * Get SPNEGO keytab Key from configuration
1344 *
1345 * @param conf
1346 * Configuration
1347 * @param defaultKey
1348 * @return DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY if the key is not empty
1349 * else return defaultKey
1350 */
1351 public static String getSpnegoKeytabKey(Configuration conf, String defaultKey) {
1352 String value =
1353 conf.get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY);
1354 return (value == null || value.isEmpty()) ?
1355 defaultKey : DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY;
1356 }
1357 }