001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.web;
019
020 import org.apache.hadoop.fs.*;
021 import org.apache.hadoop.fs.permission.FsPermission;
022 import org.apache.hadoop.hdfs.DFSUtil;
023 import org.apache.hadoop.hdfs.protocol.*;
024 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
025 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
026 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
027 import org.apache.hadoop.hdfs.server.namenode.INodeId;
028 import org.apache.hadoop.ipc.RemoteException;
029 import org.apache.hadoop.security.token.Token;
030 import org.apache.hadoop.security.token.TokenIdentifier;
031 import org.apache.hadoop.util.DataChecksum;
032 import org.apache.hadoop.util.StringUtils;
033 import org.mortbay.util.ajax.JSON;
034
035 import java.io.ByteArrayInputStream;
036 import java.io.DataInputStream;
037 import java.io.IOException;
038 import java.util.*;
039
040 /** JSON Utilities */
041 public class JsonUtil {
042 private static final Object[] EMPTY_OBJECT_ARRAY = {};
043 private static final DatanodeInfo[] EMPTY_DATANODE_INFO_ARRAY = {};
044
045 /** Convert a token object to a Json string. */
046 public static String toJsonString(final Token<? extends TokenIdentifier> token
047 ) throws IOException {
048 return toJsonString(Token.class, toJsonMap(token));
049 }
050
051 private static Map<String, Object> toJsonMap(
052 final Token<? extends TokenIdentifier> token) throws IOException {
053 if (token == null) {
054 return null;
055 }
056
057 final Map<String, Object> m = new TreeMap<String, Object>();
058 m.put("urlString", token.encodeToUrlString());
059 return m;
060 }
061
062 /** Convert a Json map to a Token. */
063 public static Token<? extends TokenIdentifier> toToken(
064 final Map<?, ?> m) throws IOException {
065 if (m == null) {
066 return null;
067 }
068
069 final Token<DelegationTokenIdentifier> token
070 = new Token<DelegationTokenIdentifier>();
071 token.decodeFromUrlString((String)m.get("urlString"));
072 return token;
073 }
074
075 /** Convert a Json map to a Token of DelegationTokenIdentifier. */
076 @SuppressWarnings("unchecked")
077 public static Token<DelegationTokenIdentifier> toDelegationToken(
078 final Map<?, ?> json) throws IOException {
079 final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName());
080 return (Token<DelegationTokenIdentifier>)toToken(m);
081 }
082
083 /** Convert a Json map to a Token of BlockTokenIdentifier. */
084 @SuppressWarnings("unchecked")
085 private static Token<BlockTokenIdentifier> toBlockToken(
086 final Map<?, ?> m) throws IOException {
087 return (Token<BlockTokenIdentifier>)toToken(m);
088 }
089
090 /** Convert a Token[] to a JSON array. */
091 private static Object[] toJsonArray(final Token<? extends TokenIdentifier>[] array
092 ) throws IOException {
093 if (array == null) {
094 return null;
095 } else if (array.length == 0) {
096 return EMPTY_OBJECT_ARRAY;
097 } else {
098 final Object[] a = new Object[array.length];
099 for(int i = 0; i < array.length; i++) {
100 a[i] = toJsonMap(array[i]);
101 }
102 return a;
103 }
104 }
105
106 /** Convert a token object to a JSON string. */
107 public static String toJsonString(final Token<? extends TokenIdentifier>[] tokens
108 ) throws IOException {
109 if (tokens == null) {
110 return null;
111 }
112
113 final Map<String, Object> m = new TreeMap<String, Object>();
114 m.put(Token.class.getSimpleName(), toJsonArray(tokens));
115 return toJsonString(Token.class.getSimpleName() + "s", m);
116 }
117
118 /** Convert an Object[] to a List<Token<?>>. */
119 private static List<Token<?>> toTokenList(final Object[] objects) throws IOException {
120 if (objects == null) {
121 return null;
122 } else if (objects.length == 0) {
123 return Collections.emptyList();
124 } else {
125 final List<Token<?>> list = new ArrayList<Token<?>>(objects.length);
126 for(int i = 0; i < objects.length; i++) {
127 list.add(toToken((Map<?, ?>)objects[i]));
128 }
129 return list;
130 }
131 }
132
133 /** Convert a JSON map to a List<Token<?>>. */
134 public static List<Token<?>> toTokenList(final Map<?, ?> json) throws IOException {
135 if (json == null) {
136 return null;
137 }
138
139 final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName() + "s");
140 return toTokenList((Object[])m.get(Token.class.getSimpleName()));
141 }
142
143 /** Convert an exception object to a Json string. */
144 public static String toJsonString(final Exception e) {
145 final Map<String, Object> m = new TreeMap<String, Object>();
146 m.put("exception", e.getClass().getSimpleName());
147 m.put("message", e.getMessage());
148 m.put("javaClassName", e.getClass().getName());
149 return toJsonString(RemoteException.class, m);
150 }
151
152 /** Convert a Json map to a RemoteException. */
153 public static RemoteException toRemoteException(final Map<?, ?> json) {
154 final Map<?, ?> m = (Map<?, ?>)json.get(RemoteException.class.getSimpleName());
155 final String message = (String)m.get("message");
156 final String javaClassName = (String)m.get("javaClassName");
157 return new RemoteException(javaClassName, message);
158 }
159
160 private static String toJsonString(final Class<?> clazz, final Object value) {
161 return toJsonString(clazz.getSimpleName(), value);
162 }
163
164 /** Convert a key-value pair to a Json string. */
165 public static String toJsonString(final String key, final Object value) {
166 final Map<String, Object> m = new TreeMap<String, Object>();
167 m.put(key, value);
168 return JSON.toString(m);
169 }
170
171 /** Convert a FsPermission object to a string. */
172 private static String toString(final FsPermission permission) {
173 return String.format("%o", permission.toShort());
174 }
175
176 /** Convert a string to a FsPermission object. */
177 private static FsPermission toFsPermission(final String s) {
178 return new FsPermission(Short.parseShort(s, 8));
179 }
180
181 static enum PathType {
182 FILE, DIRECTORY, SYMLINK;
183
184 static PathType valueOf(HdfsFileStatus status) {
185 return status.isDir()? DIRECTORY: status.isSymlink()? SYMLINK: FILE;
186 }
187 }
188
189 /** Convert a HdfsFileStatus object to a Json string. */
190 public static String toJsonString(final HdfsFileStatus status,
191 boolean includeType) {
192 if (status == null) {
193 return null;
194 }
195 final Map<String, Object> m = new TreeMap<String, Object>();
196 m.put("pathSuffix", status.getLocalName());
197 m.put("type", PathType.valueOf(status));
198 if (status.isSymlink()) {
199 m.put("symlink", status.getSymlink());
200 }
201
202 m.put("length", status.getLen());
203 m.put("owner", status.getOwner());
204 m.put("group", status.getGroup());
205 m.put("permission", toString(status.getPermission()));
206 m.put("accessTime", status.getAccessTime());
207 m.put("modificationTime", status.getModificationTime());
208 m.put("blockSize", status.getBlockSize());
209 m.put("replication", status.getReplication());
210 m.put("fileId", status.getFileId());
211 m.put("childrenNum", status.getChildrenNum());
212 return includeType ? toJsonString(FileStatus.class, m): JSON.toString(m);
213 }
214
215 /** Convert a Json map to a HdfsFileStatus object. */
216 public static HdfsFileStatus toFileStatus(final Map<?, ?> json, boolean includesType) {
217 if (json == null) {
218 return null;
219 }
220
221 final Map<?, ?> m = includesType ?
222 (Map<?, ?>)json.get(FileStatus.class.getSimpleName()) : json;
223 final String localName = (String) m.get("pathSuffix");
224 final PathType type = PathType.valueOf((String) m.get("type"));
225 final byte[] symlink = type != PathType.SYMLINK? null
226 : DFSUtil.string2Bytes((String)m.get("symlink"));
227
228 final long len = (Long) m.get("length");
229 final String owner = (String) m.get("owner");
230 final String group = (String) m.get("group");
231 final FsPermission permission = toFsPermission((String) m.get("permission"));
232 final long aTime = (Long) m.get("accessTime");
233 final long mTime = (Long) m.get("modificationTime");
234 final long blockSize = (Long) m.get("blockSize");
235 final short replication = (short) (long) (Long) m.get("replication");
236 final long fileId = m.containsKey("fileId") ? (Long) m.get("fileId")
237 : INodeId.GRANDFATHER_INODE_ID;
238 Long childrenNumLong = (Long) m.get("childrenNum");
239 final int childrenNum = (childrenNumLong == null) ? -1
240 : childrenNumLong.intValue();
241 return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication,
242 blockSize, mTime, aTime, permission, owner, group,
243 symlink, DFSUtil.string2Bytes(localName), fileId, childrenNum);
244 }
245
246 /** Convert an ExtendedBlock to a Json map. */
247 private static Map<String, Object> toJsonMap(final ExtendedBlock extendedblock) {
248 if (extendedblock == null) {
249 return null;
250 }
251
252 final Map<String, Object> m = new TreeMap<String, Object>();
253 m.put("blockPoolId", extendedblock.getBlockPoolId());
254 m.put("blockId", extendedblock.getBlockId());
255 m.put("numBytes", extendedblock.getNumBytes());
256 m.put("generationStamp", extendedblock.getGenerationStamp());
257 return m;
258 }
259
260 /** Convert a Json map to an ExtendedBlock object. */
261 private static ExtendedBlock toExtendedBlock(final Map<?, ?> m) {
262 if (m == null) {
263 return null;
264 }
265
266 final String blockPoolId = (String)m.get("blockPoolId");
267 final long blockId = (Long)m.get("blockId");
268 final long numBytes = (Long)m.get("numBytes");
269 final long generationStamp = (Long)m.get("generationStamp");
270 return new ExtendedBlock(blockPoolId, blockId, numBytes, generationStamp);
271 }
272
273 /** Convert a DatanodeInfo to a Json map. */
274 static Map<String, Object> toJsonMap(final DatanodeInfo datanodeinfo) {
275 if (datanodeinfo == null) {
276 return null;
277 }
278
279 final Map<String, Object> m = new TreeMap<String, Object>();
280 m.put("ipAddr", datanodeinfo.getIpAddr());
281 // 'name' is equivalent to ipAddr:xferPort. Older clients (1.x, 0.23.x)
282 // expects this instead of the two fields.
283 m.put("name", datanodeinfo.getXferAddr());
284 m.put("hostName", datanodeinfo.getHostName());
285 m.put("storageID", datanodeinfo.getStorageID());
286 m.put("xferPort", datanodeinfo.getXferPort());
287 m.put("infoPort", datanodeinfo.getInfoPort());
288 m.put("infoSecurePort", datanodeinfo.getInfoSecurePort());
289 m.put("ipcPort", datanodeinfo.getIpcPort());
290
291 m.put("capacity", datanodeinfo.getCapacity());
292 m.put("dfsUsed", datanodeinfo.getDfsUsed());
293 m.put("remaining", datanodeinfo.getRemaining());
294 m.put("blockPoolUsed", datanodeinfo.getBlockPoolUsed());
295 m.put("lastUpdate", datanodeinfo.getLastUpdate());
296 m.put("xceiverCount", datanodeinfo.getXceiverCount());
297 m.put("networkLocation", datanodeinfo.getNetworkLocation());
298 m.put("adminState", datanodeinfo.getAdminState().name());
299 return m;
300 }
301
302 /** Convert a Json map to an DatanodeInfo object. */
303 static DatanodeInfo toDatanodeInfo(final Map<?, ?> m)
304 throws IOException {
305 if (m == null) {
306 return null;
307 }
308
309 Object infoSecurePort = m.get("infoSecurePort");
310 if (infoSecurePort == null) {
311 infoSecurePort = 0l; // same as the default value in hdfs.proto
312 }
313
314 // ipAddr and xferPort are the critical fields for accessing data.
315 // If any one of the two is missing, an exception needs to be thrown.
316
317 // Handle the case of old servers (1.x, 0.23.x) sending 'name' instead
318 // of ipAddr and xferPort.
319 Object tmpValue = m.get("ipAddr");
320 String ipAddr = (tmpValue == null) ? null : (String)tmpValue;
321 tmpValue = m.get("xferPort");
322 int xferPort = (tmpValue == null) ? -1 : (int)(long)(Long)tmpValue;
323 if (ipAddr == null) {
324 tmpValue = m.get("name");
325 if (tmpValue != null) {
326 String name = (String)tmpValue;
327 int colonIdx = name.indexOf(':');
328 if (colonIdx > 0) {
329 ipAddr = name.substring(0, colonIdx);
330 xferPort = Integer.parseInt(name.substring(colonIdx +1));
331 } else {
332 throw new IOException(
333 "Invalid value in server response: name=[" + name + "]");
334 }
335 } else {
336 throw new IOException(
337 "Missing both 'ipAddr' and 'name' in server response.");
338 }
339 // ipAddr is non-null & non-empty string at this point.
340 }
341
342 // Check the validity of xferPort.
343 if (xferPort == -1) {
344 throw new IOException(
345 "Invalid or missing 'xferPort' in server response.");
346 }
347
348 return new DatanodeInfo(
349 ipAddr,
350 (String)m.get("hostName"),
351 (String)m.get("storageID"),
352 xferPort,
353 (int)(long)(Long)m.get("infoPort"),
354 (int)(long)(Long)infoSecurePort,
355 (int)(long)(Long)m.get("ipcPort"),
356
357 (Long)m.get("capacity"),
358 (Long)m.get("dfsUsed"),
359 (Long)m.get("remaining"),
360 (Long)m.get("blockPoolUsed"),
361 (Long)m.get("lastUpdate"),
362 (int)(long)(Long)m.get("xceiverCount"),
363 (String)m.get("networkLocation"),
364 AdminStates.valueOf((String)m.get("adminState")));
365 }
366
367 /** Convert a DatanodeInfo[] to a Json array. */
368 private static Object[] toJsonArray(final DatanodeInfo[] array) {
369 if (array == null) {
370 return null;
371 } else if (array.length == 0) {
372 return EMPTY_OBJECT_ARRAY;
373 } else {
374 final Object[] a = new Object[array.length];
375 for(int i = 0; i < array.length; i++) {
376 a[i] = toJsonMap(array[i]);
377 }
378 return a;
379 }
380 }
381
382 /** Convert an Object[] to a DatanodeInfo[]. */
383 private static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects)
384 throws IOException {
385 if (objects == null) {
386 return null;
387 } else if (objects.length == 0) {
388 return EMPTY_DATANODE_INFO_ARRAY;
389 } else {
390 final DatanodeInfo[] array = new DatanodeInfo[objects.length];
391 for(int i = 0; i < array.length; i++) {
392 array[i] = toDatanodeInfo((Map<?, ?>) objects[i]);
393 }
394 return array;
395 }
396 }
397
398 /** Convert a LocatedBlock to a Json map. */
399 private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock
400 ) throws IOException {
401 if (locatedblock == null) {
402 return null;
403 }
404
405 final Map<String, Object> m = new TreeMap<String, Object>();
406 m.put("blockToken", toJsonMap(locatedblock.getBlockToken()));
407 m.put("isCorrupt", locatedblock.isCorrupt());
408 m.put("startOffset", locatedblock.getStartOffset());
409 m.put("block", toJsonMap(locatedblock.getBlock()));
410 m.put("locations", toJsonArray(locatedblock.getLocations()));
411 return m;
412 }
413
414 /** Convert a Json map to LocatedBlock. */
415 private static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException {
416 if (m == null) {
417 return null;
418 }
419
420 final ExtendedBlock b = toExtendedBlock((Map<?, ?>)m.get("block"));
421 final DatanodeInfo[] locations = toDatanodeInfoArray(
422 (Object[])m.get("locations"));
423 final long startOffset = (Long)m.get("startOffset");
424 final boolean isCorrupt = (Boolean)m.get("isCorrupt");
425
426 final LocatedBlock locatedblock = new LocatedBlock(b, locations, startOffset, isCorrupt);
427 locatedblock.setBlockToken(toBlockToken((Map<?, ?>)m.get("blockToken")));
428 return locatedblock;
429 }
430
431 /** Convert a LocatedBlock[] to a Json array. */
432 private static Object[] toJsonArray(final List<LocatedBlock> array
433 ) throws IOException {
434 if (array == null) {
435 return null;
436 } else if (array.size() == 0) {
437 return EMPTY_OBJECT_ARRAY;
438 } else {
439 final Object[] a = new Object[array.size()];
440 for(int i = 0; i < array.size(); i++) {
441 a[i] = toJsonMap(array.get(i));
442 }
443 return a;
444 }
445 }
446
447 /** Convert an Object[] to a List of LocatedBlock. */
448 private static List<LocatedBlock> toLocatedBlockList(final Object[] objects
449 ) throws IOException {
450 if (objects == null) {
451 return null;
452 } else if (objects.length == 0) {
453 return Collections.emptyList();
454 } else {
455 final List<LocatedBlock> list = new ArrayList<LocatedBlock>(objects.length);
456 for(int i = 0; i < objects.length; i++) {
457 list.add(toLocatedBlock((Map<?, ?>)objects[i]));
458 }
459 return list;
460 }
461 }
462
463 /** Convert LocatedBlocks to a Json string. */
464 public static String toJsonString(final LocatedBlocks locatedblocks
465 ) throws IOException {
466 if (locatedblocks == null) {
467 return null;
468 }
469
470 final Map<String, Object> m = new TreeMap<String, Object>();
471 m.put("fileLength", locatedblocks.getFileLength());
472 m.put("isUnderConstruction", locatedblocks.isUnderConstruction());
473
474 m.put("locatedBlocks", toJsonArray(locatedblocks.getLocatedBlocks()));
475 m.put("lastLocatedBlock", toJsonMap(locatedblocks.getLastLocatedBlock()));
476 m.put("isLastBlockComplete", locatedblocks.isLastBlockComplete());
477 return toJsonString(LocatedBlocks.class, m);
478 }
479
480 /** Convert a Json map to LocatedBlock. */
481 public static LocatedBlocks toLocatedBlocks(final Map<?, ?> json
482 ) throws IOException {
483 if (json == null) {
484 return null;
485 }
486
487 final Map<?, ?> m = (Map<?, ?>)json.get(LocatedBlocks.class.getSimpleName());
488 final long fileLength = (Long)m.get("fileLength");
489 final boolean isUnderConstruction = (Boolean)m.get("isUnderConstruction");
490 final List<LocatedBlock> locatedBlocks = toLocatedBlockList(
491 (Object[])m.get("locatedBlocks"));
492 final LocatedBlock lastLocatedBlock = toLocatedBlock(
493 (Map<?, ?>)m.get("lastLocatedBlock"));
494 final boolean isLastBlockComplete = (Boolean)m.get("isLastBlockComplete");
495 return new LocatedBlocks(fileLength, isUnderConstruction, locatedBlocks,
496 lastLocatedBlock, isLastBlockComplete);
497 }
498
499 /** Convert a ContentSummary to a Json string. */
500 public static String toJsonString(final ContentSummary contentsummary) {
501 if (contentsummary == null) {
502 return null;
503 }
504
505 final Map<String, Object> m = new TreeMap<String, Object>();
506 m.put("length", contentsummary.getLength());
507 m.put("fileCount", contentsummary.getFileCount());
508 m.put("directoryCount", contentsummary.getDirectoryCount());
509 m.put("quota", contentsummary.getQuota());
510 m.put("spaceConsumed", contentsummary.getSpaceConsumed());
511 m.put("spaceQuota", contentsummary.getSpaceQuota());
512 return toJsonString(ContentSummary.class, m);
513 }
514
515 /** Convert a Json map to a ContentSummary. */
516 public static ContentSummary toContentSummary(final Map<?, ?> json) {
517 if (json == null) {
518 return null;
519 }
520
521 final Map<?, ?> m = (Map<?, ?>)json.get(ContentSummary.class.getSimpleName());
522 final long length = (Long)m.get("length");
523 final long fileCount = (Long)m.get("fileCount");
524 final long directoryCount = (Long)m.get("directoryCount");
525 final long quota = (Long)m.get("quota");
526 final long spaceConsumed = (Long)m.get("spaceConsumed");
527 final long spaceQuota = (Long)m.get("spaceQuota");
528
529 return new ContentSummary(length, fileCount, directoryCount,
530 quota, spaceConsumed, spaceQuota);
531 }
532
533 /** Convert a MD5MD5CRC32FileChecksum to a Json string. */
534 public static String toJsonString(final MD5MD5CRC32FileChecksum checksum) {
535 if (checksum == null) {
536 return null;
537 }
538
539 final Map<String, Object> m = new TreeMap<String, Object>();
540 m.put("algorithm", checksum.getAlgorithmName());
541 m.put("length", checksum.getLength());
542 m.put("bytes", StringUtils.byteToHexString(checksum.getBytes()));
543 return toJsonString(FileChecksum.class, m);
544 }
545
546 /** Convert a Json map to a MD5MD5CRC32FileChecksum. */
547 public static MD5MD5CRC32FileChecksum toMD5MD5CRC32FileChecksum(
548 final Map<?, ?> json) throws IOException {
549 if (json == null) {
550 return null;
551 }
552
553 final Map<?, ?> m = (Map<?, ?>)json.get(FileChecksum.class.getSimpleName());
554 final String algorithm = (String)m.get("algorithm");
555 final int length = (int)(long)(Long)m.get("length");
556 final byte[] bytes = StringUtils.hexStringToByte((String)m.get("bytes"));
557
558 final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
559 final DataChecksum.Type crcType =
560 MD5MD5CRC32FileChecksum.getCrcTypeFromAlgorithmName(algorithm);
561 final MD5MD5CRC32FileChecksum checksum;
562
563 // Recreate what DFSClient would have returned.
564 switch(crcType) {
565 case CRC32:
566 checksum = new MD5MD5CRC32GzipFileChecksum();
567 break;
568 case CRC32C:
569 checksum = new MD5MD5CRC32CastagnoliFileChecksum();
570 break;
571 default:
572 throw new IOException("Unknown algorithm: " + algorithm);
573 }
574 checksum.readFields(in);
575
576 //check algorithm name
577 if (!checksum.getAlgorithmName().equals(algorithm)) {
578 throw new IOException("Algorithm not matched. Expected " + algorithm
579 + ", Received " + checksum.getAlgorithmName());
580 }
581 //check length
582 if (length != checksum.getLength()) {
583 throw new IOException("Length not matched: length=" + length
584 + ", checksum.getLength()=" + checksum.getLength());
585 }
586
587 return checksum;
588 }
589 }