001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.web;
020
021 import java.io.BufferedOutputStream;
022 import java.io.FileNotFoundException;
023 import java.io.IOException;
024 import java.io.InputStream;
025 import java.io.InputStreamReader;
026 import java.net.HttpURLConnection;
027 import java.net.InetSocketAddress;
028 import java.net.MalformedURLException;
029 import java.net.URI;
030 import java.net.URISyntaxException;
031 import java.net.URL;
032 import java.security.PrivilegedExceptionAction;
033 import java.util.Collection;
034 import java.util.List;
035 import java.util.Map;
036 import java.util.StringTokenizer;
037
038 import javax.ws.rs.core.MediaType;
039
040 import org.apache.commons.logging.Log;
041 import org.apache.commons.logging.LogFactory;
042 import org.apache.hadoop.conf.Configuration;
043 import org.apache.hadoop.fs.BlockLocation;
044 import org.apache.hadoop.fs.ContentSummary;
045 import org.apache.hadoop.fs.DelegationTokenRenewer;
046 import org.apache.hadoop.fs.FSDataInputStream;
047 import org.apache.hadoop.fs.FSDataOutputStream;
048 import org.apache.hadoop.fs.FileStatus;
049 import org.apache.hadoop.fs.FileSystem;
050 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
051 import org.apache.hadoop.fs.Options;
052 import org.apache.hadoop.fs.Path;
053 import org.apache.hadoop.fs.permission.FsPermission;
054 import org.apache.hadoop.hdfs.ByteRangeInputStream;
055 import org.apache.hadoop.hdfs.DFSConfigKeys;
056 import org.apache.hadoop.hdfs.DFSUtil;
057 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
058 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
059 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
060 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
061 import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
062 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
063 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
064 import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam;
065 import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
066 import org.apache.hadoop.hdfs.web.resources.DelegationParam;
067 import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
068 import org.apache.hadoop.hdfs.web.resources.DestinationParam;
069 import org.apache.hadoop.hdfs.web.resources.DoAsParam;
070 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
071 import org.apache.hadoop.hdfs.web.resources.GroupParam;
072 import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
073 import org.apache.hadoop.hdfs.web.resources.LengthParam;
074 import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
075 import org.apache.hadoop.hdfs.web.resources.OffsetParam;
076 import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
077 import org.apache.hadoop.hdfs.web.resources.OwnerParam;
078 import org.apache.hadoop.hdfs.web.resources.Param;
079 import org.apache.hadoop.hdfs.web.resources.PermissionParam;
080 import org.apache.hadoop.hdfs.web.resources.PostOpParam;
081 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
082 import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
083 import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
084 import org.apache.hadoop.hdfs.web.resources.RenewerParam;
085 import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
086 import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
087 import org.apache.hadoop.hdfs.web.resources.UserParam;
088 import org.apache.hadoop.io.Text;
089 import org.apache.hadoop.io.retry.RetryPolicy;
090 import org.apache.hadoop.io.retry.RetryUtils;
091 import org.apache.hadoop.ipc.RemoteException;
092 import org.apache.hadoop.net.NetUtils;
093 import org.apache.hadoop.security.SecurityUtil;
094 import org.apache.hadoop.security.UserGroupInformation;
095 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
096 import org.apache.hadoop.security.authentication.client.AuthenticationException;
097 import org.apache.hadoop.security.authorize.AuthorizationException;
098 import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
099 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
100 import org.apache.hadoop.security.token.Token;
101 import org.apache.hadoop.security.token.TokenIdentifier;
102 import org.apache.hadoop.security.token.TokenRenewer;
103 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
104 import org.apache.hadoop.util.Progressable;
105 import org.mortbay.util.ajax.JSON;
106
107 import com.google.common.annotations.VisibleForTesting;
108 import com.google.common.base.Charsets;
109 import com.google.common.collect.Lists;
110
111 /** A FileSystem for HDFS over the web. */
112 public class WebHdfsFileSystem extends FileSystem
113 implements DelegationTokenRenewer.Renewable {
114 public static final Log LOG = LogFactory.getLog(WebHdfsFileSystem.class);
115 /** File System URI: {SCHEME}://namenode:port/path/to/file */
116 public static final String SCHEME = "webhdfs";
117 /** WebHdfs version. */
118 public static final int VERSION = 1;
119 /** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
120 public static final String PATH_PREFIX = "/" + SCHEME + "/v" + VERSION;
121
122 /** SPNEGO authenticator */
123 private static final KerberosUgiAuthenticator AUTH = new KerberosUgiAuthenticator();
124 /** Configures connections for AuthenticatedURL */
125 private static final ConnectionConfigurator CONN_CONFIGURATOR =
126 new ConnectionConfigurator() {
127 @Override
128 public HttpURLConnection configure(HttpURLConnection conn)
129 throws IOException {
130 URLUtils.setTimeouts(conn);
131 return conn;
132 }
133 };
134 /** Delegation token kind */
135 public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
136 /** Token selector */
137 public static final WebHdfsDelegationTokenSelector DT_SELECTOR
138 = new WebHdfsDelegationTokenSelector();
139
140 private DelegationTokenRenewer dtRenewer = null;
141 @VisibleForTesting
142 DelegationTokenRenewer.RenewAction<?> action;
143
144 @VisibleForTesting
145 protected synchronized void addRenewAction(final WebHdfsFileSystem webhdfs) {
146 if (dtRenewer == null) {
147 dtRenewer = DelegationTokenRenewer.getInstance();
148 }
149
150 action = dtRenewer.addRenewAction(webhdfs);
151 }
152
153 /** Is WebHDFS enabled in conf? */
154 public static boolean isEnabled(final Configuration conf, final Log log) {
155 final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
156 DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT);
157 log.info(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY + " = " + b);
158 return b;
159 }
160
161 private UserGroupInformation ugi;
162 private InetSocketAddress nnAddr;
163 private URI uri;
164 private boolean hasInitedToken;
165 private Token<?> delegationToken;
166 private RetryPolicy retryPolicy = null;
167 private Path workingDir;
168
169 /**
170 * Return the protocol scheme for the FileSystem.
171 * <p/>
172 *
173 * @return <code>webhdfs</code>
174 */
175 @Override
176 public String getScheme() {
177 return "webhdfs";
178 }
179
180 @Override
181 public synchronized void initialize(URI uri, Configuration conf
182 ) throws IOException {
183 super.initialize(uri, conf);
184 setConf(conf);
185 /** set user pattern based on configuration file */
186 UserParam.setUserPattern(conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
187
188 ugi = UserGroupInformation.getCurrentUser();
189 try {
190 this.uri = new URI(uri.getScheme(), uri.getAuthority(), null, null, null);
191 } catch (URISyntaxException e) {
192 throw new IllegalArgumentException(e);
193 }
194 this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort());
195 this.retryPolicy =
196 RetryUtils.getDefaultRetryPolicy(
197 conf,
198 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY,
199 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
200 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY,
201 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
202 SafeModeException.class);
203 this.workingDir = getHomeDirectory();
204
205 if (UserGroupInformation.isSecurityEnabled()) {
206 initDelegationToken();
207 }
208 }
209
210 protected void initDelegationToken() throws IOException {
211 // look for webhdfs token, then try hdfs
212 Token<?> token = selectDelegationToken(ugi);
213 if (token != null) {
214 LOG.debug("Found existing DT for " + token.getService());
215 setDelegationToken(token);
216 hasInitedToken = true;
217 }
218 }
219
220 protected synchronized Token<?> getDelegationToken() throws IOException {
221 // we haven't inited yet, or we used to have a token but it expired
222 if (!hasInitedToken || (action != null && !action.isValid())) {
223 //since we don't already have a token, go get one
224 Token<?> token = getDelegationToken(null);
225 // security might be disabled
226 if (token != null) {
227 setDelegationToken(token);
228 addRenewAction(this);
229 LOG.debug("Created new DT for " + token.getService());
230 }
231 hasInitedToken = true;
232 }
233 return delegationToken;
234 }
235
236 protected Token<DelegationTokenIdentifier> selectDelegationToken(
237 UserGroupInformation ugi) {
238 return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
239 }
240
241 @Override
242 protected int getDefaultPort() {
243 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
244 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
245 }
246
247 @Override
248 public URI getUri() {
249 return this.uri;
250 }
251
252 @Override
253 protected URI canonicalizeUri(URI uri) {
254 return NetUtils.getCanonicalUri(uri, getDefaultPort());
255 }
256
257 /** @return the home directory. */
258 public static String getHomeDirectoryString(final UserGroupInformation ugi) {
259 return "/user/" + ugi.getShortUserName();
260 }
261
262 @Override
263 public Path getHomeDirectory() {
264 return makeQualified(new Path(getHomeDirectoryString(ugi)));
265 }
266
267 @Override
268 public synchronized Path getWorkingDirectory() {
269 return workingDir;
270 }
271
272 @Override
273 public synchronized void setWorkingDirectory(final Path dir) {
274 String result = makeAbsolute(dir).toUri().getPath();
275 if (!DFSUtil.isValidName(result)) {
276 throw new IllegalArgumentException("Invalid DFS directory name " +
277 result);
278 }
279 workingDir = makeAbsolute(dir);
280 }
281
282 private Path makeAbsolute(Path f) {
283 return f.isAbsolute()? f: new Path(workingDir, f);
284 }
285
286 static Map<?, ?> jsonParse(final HttpURLConnection c, final boolean useErrorStream
287 ) throws IOException {
288 if (c.getContentLength() == 0) {
289 return null;
290 }
291 final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream();
292 if (in == null) {
293 throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
294 }
295 final String contentType = c.getContentType();
296 if (contentType != null) {
297 final MediaType parsed = MediaType.valueOf(contentType);
298 if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
299 throw new IOException("Content-Type \"" + contentType
300 + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
301 + "\" (parsed=\"" + parsed + "\")");
302 }
303 }
304 return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
305 }
306
307 private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
308 final HttpURLConnection conn, boolean unwrapException) throws IOException {
309 final int code = conn.getResponseCode();
310 if (code != op.getExpectedHttpResponseCode()) {
311 final Map<?, ?> m;
312 try {
313 m = jsonParse(conn, true);
314 } catch(Exception e) {
315 throw new IOException("Unexpected HTTP response: code=" + code + " != "
316 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
317 + ", message=" + conn.getResponseMessage(), e);
318 }
319
320 if (m == null) {
321 throw new IOException("Unexpected HTTP response: code=" + code + " != "
322 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
323 + ", message=" + conn.getResponseMessage());
324 } else if (m.get(RemoteException.class.getSimpleName()) == null) {
325 return m;
326 }
327
328 final RemoteException re = JsonUtil.toRemoteException(m);
329 throw unwrapException? toIOException(re): re;
330 }
331 return null;
332 }
333
334 /**
335 * Covert an exception to an IOException.
336 *
337 * For a non-IOException, wrap it with IOException.
338 * For a RemoteException, unwrap it.
339 * For an IOException which is not a RemoteException, return it.
340 */
341 private static IOException toIOException(Exception e) {
342 if (!(e instanceof IOException)) {
343 return new IOException(e);
344 }
345
346 final IOException ioe = (IOException)e;
347 if (!(ioe instanceof RemoteException)) {
348 return ioe;
349 }
350
351 return ((RemoteException)ioe).unwrapRemoteException();
352 }
353
354 /**
355 * Return a URL pointing to given path on the namenode.
356 *
357 * @param path to obtain the URL for
358 * @param query string to append to the path
359 * @return namenode URL referring to the given path
360 * @throws IOException on error constructing the URL
361 */
362 private URL getNamenodeURL(String path, String query) throws IOException {
363 final URL url = new URL("http", nnAddr.getHostName(),
364 nnAddr.getPort(), path + '?' + query);
365 if (LOG.isTraceEnabled()) {
366 LOG.trace("url=" + url);
367 }
368 return url;
369 }
370
371 Param<?,?>[] getAuthParameters(final HttpOpParam.Op op) throws IOException {
372 List<Param<?,?>> authParams = Lists.newArrayList();
373 // Skip adding delegation token for token operations because these
374 // operations require authentication.
375 Token<?> token = null;
376 if (UserGroupInformation.isSecurityEnabled() && !op.getRequireAuth()) {
377 token = getDelegationToken();
378 }
379 if (token != null) {
380 authParams.add(new DelegationParam(token.encodeToUrlString()));
381 } else {
382 UserGroupInformation userUgi = ugi;
383 UserGroupInformation realUgi = userUgi.getRealUser();
384 if (realUgi != null) { // proxy user
385 authParams.add(new DoAsParam(userUgi.getShortUserName()));
386 userUgi = realUgi;
387 }
388 authParams.add(new UserParam(userUgi.getShortUserName()));
389 }
390 return authParams.toArray(new Param<?,?>[0]);
391 }
392
393 URL toUrl(final HttpOpParam.Op op, final Path fspath,
394 final Param<?,?>... parameters) throws IOException {
395 //initialize URI path and query
396 final String path = PATH_PREFIX
397 + (fspath == null? "/": makeQualified(fspath).toUri().getRawPath());
398 final String query = op.toQueryString()
399 + Param.toSortedString("&", getAuthParameters(op))
400 + Param.toSortedString("&", parameters);
401 final URL url = getNamenodeURL(path, query);
402 if (LOG.isTraceEnabled()) {
403 LOG.trace("url=" + url);
404 }
405 return url;
406 }
407
408 /**
409 * Run a http operation.
410 * Connect to the http server, validate response, and obtain the JSON output.
411 *
412 * @param op http operation
413 * @param fspath file system path
414 * @param parameters parameters for the operation
415 * @return a JSON object, e.g. Object[], Map<?, ?>, etc.
416 * @throws IOException
417 */
418 private Map<?, ?> run(final HttpOpParam.Op op, final Path fspath,
419 final Param<?,?>... parameters) throws IOException {
420 return new Runner(op, fspath, parameters).run().json;
421 }
422
423 /**
424 * This class is for initialing a HTTP connection, connecting to server,
425 * obtaining a response, and also handling retry on failures.
426 */
427 class Runner {
428 private final HttpOpParam.Op op;
429 private final URL url;
430 private final boolean redirected;
431
432 private boolean checkRetry;
433 private HttpURLConnection conn = null;
434 private Map<?, ?> json = null;
435
436 Runner(final HttpOpParam.Op op, final URL url, final boolean redirected) {
437 this.op = op;
438 this.url = url;
439 this.redirected = redirected;
440 }
441
442 Runner(final HttpOpParam.Op op, final Path fspath,
443 final Param<?,?>... parameters) throws IOException {
444 this(op, toUrl(op, fspath, parameters), false);
445 }
446
447 Runner(final HttpOpParam.Op op, final HttpURLConnection conn) {
448 this(op, null, false);
449 this.conn = conn;
450 }
451
452 private HttpURLConnection getHttpUrlConnection(final URL url)
453 throws IOException, AuthenticationException {
454 UserGroupInformation connectUgi = ugi.getRealUser();
455 if (connectUgi == null) {
456 connectUgi = ugi;
457 }
458 try {
459 return connectUgi.doAs(
460 new PrivilegedExceptionAction<HttpURLConnection>() {
461 @Override
462 public HttpURLConnection run() throws IOException {
463 return openHttpUrlConnection(url);
464 }
465 });
466 } catch (IOException ioe) {
467 Throwable cause = ioe.getCause();
468 if (cause != null && cause instanceof AuthenticationException) {
469 throw (AuthenticationException)cause;
470 }
471 throw ioe;
472 } catch (InterruptedException e) {
473 throw new IOException(e);
474 }
475 }
476
477 private HttpURLConnection openHttpUrlConnection(final URL url)
478 throws IOException {
479 final HttpURLConnection conn;
480 try {
481 if (op.getRequireAuth()) {
482 LOG.debug("open AuthenticatedURL connection");
483 UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
484 final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
485 conn = new AuthenticatedURL(AUTH, CONN_CONFIGURATOR).openConnection(
486 url, authToken);
487 URLUtils.setTimeouts(conn);
488 } else {
489 LOG.debug("open URL connection");
490 conn = (HttpURLConnection)URLUtils.openConnection(url);
491 }
492 } catch (AuthenticationException e) {
493 throw new IOException(e);
494 }
495 return conn;
496 }
497
498 private void init() throws IOException {
499 checkRetry = !redirected;
500 try {
501 conn = getHttpUrlConnection(url);
502 } catch(AuthenticationException ae) {
503 checkRetry = false;
504 throw new IOException("Authentication failed, url=" + url, ae);
505 }
506 }
507
508 private void connect() throws IOException {
509 connect(op.getDoOutput());
510 }
511
512 private void connect(boolean doOutput) throws IOException {
513 conn.setRequestMethod(op.getType().toString());
514 conn.setDoOutput(doOutput);
515 conn.setInstanceFollowRedirects(false);
516 conn.connect();
517 }
518
519 private void disconnect() {
520 if (conn != null) {
521 conn.disconnect();
522 conn = null;
523 }
524 }
525
526 Runner run() throws IOException {
527 for(int retry = 0; ; retry++) {
528 try {
529 init();
530 if (op.getDoOutput()) {
531 twoStepWrite();
532 } else {
533 getResponse(op != GetOpParam.Op.OPEN);
534 }
535 return this;
536 } catch(IOException ioe) {
537 shouldRetry(ioe, retry);
538 }
539 }
540 }
541
542 private void shouldRetry(final IOException ioe, final int retry
543 ) throws IOException {
544 if (checkRetry) {
545 try {
546 final RetryPolicy.RetryAction a = retryPolicy.shouldRetry(
547 ioe, retry, 0, true);
548 if (a.action == RetryPolicy.RetryAction.RetryDecision.RETRY) {
549 LOG.info("Retrying connect to namenode: " + nnAddr
550 + ". Already tried " + retry + " time(s); retry policy is "
551 + retryPolicy + ", delay " + a.delayMillis + "ms.");
552 Thread.sleep(a.delayMillis);
553 return;
554 }
555 } catch(Exception e) {
556 LOG.warn("Original exception is ", ioe);
557 throw toIOException(e);
558 }
559 }
560 throw toIOException(ioe);
561 }
562
563 /**
564 * Two-step Create/Append:
565 * Step 1) Submit a Http request with neither auto-redirect nor data.
566 * Step 2) Submit another Http request with the URL from the Location header with data.
567 *
568 * The reason of having two-step create/append is for preventing clients to
569 * send out the data before the redirect. This issue is addressed by the
570 * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
571 * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
572 * and Java 6 http client), which do not correctly implement "Expect:
573 * 100-continue". The two-step create/append is a temporary workaround for
574 * the software library bugs.
575 */
576 HttpURLConnection twoStepWrite() throws IOException {
577 //Step 1) Submit a Http request with neither auto-redirect nor data.
578 connect(false);
579 validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn, false);
580 final String redirect = conn.getHeaderField("Location");
581 disconnect();
582 checkRetry = false;
583
584 //Step 2) Submit another Http request with the URL from the Location header with data.
585 conn = (HttpURLConnection)URLUtils.openConnection(new URL(redirect));
586 conn.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM);
587 conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
588 connect();
589 return conn;
590 }
591
592 FSDataOutputStream write(final int bufferSize) throws IOException {
593 return WebHdfsFileSystem.this.write(op, conn, bufferSize);
594 }
595
596 void getResponse(boolean getJsonAndDisconnect) throws IOException {
597 try {
598 connect();
599 final int code = conn.getResponseCode();
600 if (!redirected && op.getRedirect()
601 && code != op.getExpectedHttpResponseCode()) {
602 final String redirect = conn.getHeaderField("Location");
603 json = validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op),
604 conn, false);
605 disconnect();
606
607 checkRetry = false;
608 conn = (HttpURLConnection)URLUtils.openConnection(new URL(redirect));
609 connect();
610 }
611
612 json = validateResponse(op, conn, false);
613 if (json == null && getJsonAndDisconnect) {
614 json = jsonParse(conn, false);
615 }
616 } finally {
617 if (getJsonAndDisconnect) {
618 disconnect();
619 }
620 }
621 }
622 }
623
624 private FsPermission applyUMask(FsPermission permission) {
625 if (permission == null) {
626 permission = FsPermission.getDefault();
627 }
628 return permission.applyUMask(FsPermission.getUMask(getConf()));
629 }
630
631 private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
632 final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
633 final Map<?, ?> json = run(op, f);
634 final HdfsFileStatus status = JsonUtil.toFileStatus(json, true);
635 if (status == null) {
636 throw new FileNotFoundException("File does not exist: " + f);
637 }
638 return status;
639 }
640
641 @Override
642 public FileStatus getFileStatus(Path f) throws IOException {
643 statistics.incrementReadOps(1);
644 return makeQualified(getHdfsFileStatus(f), f);
645 }
646
647 private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
648 return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
649 f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
650 f.getPermission(), f.getOwner(), f.getGroup(),
651 f.isSymlink() ? new Path(f.getSymlink()) : null,
652 f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
653 }
654
655 @Override
656 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
657 statistics.incrementWriteOps(1);
658 final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
659 final Map<?, ?> json = run(op, f,
660 new PermissionParam(applyUMask(permission)));
661 return (Boolean)json.get("boolean");
662 }
663
664 /**
665 * Create a symlink pointing to the destination path.
666 * @see org.apache.hadoop.fs.Hdfs#createSymlink(Path, Path, boolean)
667 */
668 public void createSymlink(Path destination, Path f, boolean createParent
669 ) throws IOException {
670 statistics.incrementWriteOps(1);
671 final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
672 run(op, f, new DestinationParam(makeQualified(destination).toUri().getPath()),
673 new CreateParentParam(createParent));
674 }
675
676 @Override
677 public boolean rename(final Path src, final Path dst) throws IOException {
678 statistics.incrementWriteOps(1);
679 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
680 final Map<?, ?> json = run(op, src,
681 new DestinationParam(makeQualified(dst).toUri().getPath()));
682 return (Boolean)json.get("boolean");
683 }
684
685 @SuppressWarnings("deprecation")
686 @Override
687 public void rename(final Path src, final Path dst,
688 final Options.Rename... options) throws IOException {
689 statistics.incrementWriteOps(1);
690 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
691 run(op, src, new DestinationParam(makeQualified(dst).toUri().getPath()),
692 new RenameOptionSetParam(options));
693 }
694
695 @Override
696 public void setOwner(final Path p, final String owner, final String group
697 ) throws IOException {
698 if (owner == null && group == null) {
699 throw new IOException("owner == null && group == null");
700 }
701
702 statistics.incrementWriteOps(1);
703 final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
704 run(op, p, new OwnerParam(owner), new GroupParam(group));
705 }
706
707 @Override
708 public void setPermission(final Path p, final FsPermission permission
709 ) throws IOException {
710 statistics.incrementWriteOps(1);
711 final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
712 run(op, p, new PermissionParam(permission));
713 }
714
715 @Override
716 public boolean setReplication(final Path p, final short replication
717 ) throws IOException {
718 statistics.incrementWriteOps(1);
719 final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
720 final Map<?, ?> json = run(op, p, new ReplicationParam(replication));
721 return (Boolean)json.get("boolean");
722 }
723
724 @Override
725 public void setTimes(final Path p, final long mtime, final long atime
726 ) throws IOException {
727 statistics.incrementWriteOps(1);
728 final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
729 run(op, p, new ModificationTimeParam(mtime), new AccessTimeParam(atime));
730 }
731
732 @Override
733 public long getDefaultBlockSize() {
734 return getConf().getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
735 DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
736 }
737
738 @Override
739 public short getDefaultReplication() {
740 return (short)getConf().getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
741 DFSConfigKeys.DFS_REPLICATION_DEFAULT);
742 }
743
744 FSDataOutputStream write(final HttpOpParam.Op op,
745 final HttpURLConnection conn, final int bufferSize) throws IOException {
746 return new FSDataOutputStream(new BufferedOutputStream(
747 conn.getOutputStream(), bufferSize), statistics) {
748 @Override
749 public void close() throws IOException {
750 try {
751 super.close();
752 } finally {
753 try {
754 validateResponse(op, conn, true);
755 } finally {
756 conn.disconnect();
757 }
758 }
759 }
760 };
761 }
762
763 @Override
764 public void concat(final Path trg, final Path [] srcs) throws IOException {
765 statistics.incrementWriteOps(1);
766 final HttpOpParam.Op op = PostOpParam.Op.CONCAT;
767
768 ConcatSourcesParam param = new ConcatSourcesParam(srcs);
769 run(op, trg, param);
770 }
771
772 @Override
773 public FSDataOutputStream create(final Path f, final FsPermission permission,
774 final boolean overwrite, final int bufferSize, final short replication,
775 final long blockSize, final Progressable progress) throws IOException {
776 statistics.incrementWriteOps(1);
777
778 final HttpOpParam.Op op = PutOpParam.Op.CREATE;
779 return new Runner(op, f,
780 new PermissionParam(applyUMask(permission)),
781 new OverwriteParam(overwrite),
782 new BufferSizeParam(bufferSize),
783 new ReplicationParam(replication),
784 new BlockSizeParam(blockSize))
785 .run()
786 .write(bufferSize);
787 }
788
789 @Override
790 public FSDataOutputStream append(final Path f, final int bufferSize,
791 final Progressable progress) throws IOException {
792 statistics.incrementWriteOps(1);
793
794 final HttpOpParam.Op op = PostOpParam.Op.APPEND;
795 return new Runner(op, f, new BufferSizeParam(bufferSize))
796 .run()
797 .write(bufferSize);
798 }
799
800 @SuppressWarnings("deprecation")
801 @Override
802 public boolean delete(final Path f) throws IOException {
803 return delete(f, true);
804 }
805
806 @Override
807 public boolean delete(Path f, boolean recursive) throws IOException {
808 final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
809 final Map<?, ?> json = run(op, f, new RecursiveParam(recursive));
810 return (Boolean)json.get("boolean");
811 }
812
813 @Override
814 public FSDataInputStream open(final Path f, final int buffersize
815 ) throws IOException {
816 statistics.incrementReadOps(1);
817 final HttpOpParam.Op op = GetOpParam.Op.OPEN;
818 final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
819 return new FSDataInputStream(new OffsetUrlInputStream(
820 new OffsetUrlOpener(url), new OffsetUrlOpener(null)));
821 }
822
823 @Override
824 public void close() throws IOException {
825 super.close();
826 if (dtRenewer != null) {
827 dtRenewer.removeRenewAction(this); // blocks
828 }
829 }
830
831 class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
832 OffsetUrlOpener(final URL url) {
833 super(url);
834 }
835
836 /** Setup offset url and connect. */
837 @Override
838 protected HttpURLConnection connect(final long offset,
839 final boolean resolved) throws IOException {
840 final URL offsetUrl = offset == 0L? url
841 : new URL(url + "&" + new OffsetParam(offset));
842 return new Runner(GetOpParam.Op.OPEN, offsetUrl, resolved).run().conn;
843 }
844 }
845
846 private static final String OFFSET_PARAM_PREFIX = OffsetParam.NAME + "=";
847
848 /** Remove offset parameter, if there is any, from the url */
849 static URL removeOffsetParam(final URL url) throws MalformedURLException {
850 String query = url.getQuery();
851 if (query == null) {
852 return url;
853 }
854 final String lower = query.toLowerCase();
855 if (!lower.startsWith(OFFSET_PARAM_PREFIX)
856 && !lower.contains("&" + OFFSET_PARAM_PREFIX)) {
857 return url;
858 }
859
860 //rebuild query
861 StringBuilder b = null;
862 for(final StringTokenizer st = new StringTokenizer(query, "&");
863 st.hasMoreTokens();) {
864 final String token = st.nextToken();
865 if (!token.toLowerCase().startsWith(OFFSET_PARAM_PREFIX)) {
866 if (b == null) {
867 b = new StringBuilder("?").append(token);
868 } else {
869 b.append('&').append(token);
870 }
871 }
872 }
873 query = b == null? "": b.toString();
874
875 final String urlStr = url.toString();
876 return new URL(urlStr.substring(0, urlStr.indexOf('?')) + query);
877 }
878
879 static class OffsetUrlInputStream extends ByteRangeInputStream {
880 OffsetUrlInputStream(OffsetUrlOpener o, OffsetUrlOpener r) {
881 super(o, r);
882 }
883
884 /** Remove offset parameter before returning the resolved url. */
885 @Override
886 protected URL getResolvedUrl(final HttpURLConnection connection
887 ) throws MalformedURLException {
888 return removeOffsetParam(connection.getURL());
889 }
890 }
891
892 @Override
893 public FileStatus[] listStatus(final Path f) throws IOException {
894 statistics.incrementReadOps(1);
895
896 final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
897 final Map<?, ?> json = run(op, f);
898 final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
899 final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
900
901 //convert FileStatus
902 final FileStatus[] statuses = new FileStatus[array.length];
903 for(int i = 0; i < array.length; i++) {
904 final Map<?, ?> m = (Map<?, ?>)array[i];
905 statuses[i] = makeQualified(JsonUtil.toFileStatus(m, false), f);
906 }
907 return statuses;
908 }
909
910 @Override
911 public Token<DelegationTokenIdentifier> getDelegationToken(
912 final String renewer) throws IOException {
913 final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
914 final Map<?, ?> m = run(op, null, new RenewerParam(renewer));
915 final Token<DelegationTokenIdentifier> token = JsonUtil.toDelegationToken(m);
916 SecurityUtil.setTokenService(token, nnAddr);
917 return token;
918 }
919
920 @Override
921 public Token<?> getRenewToken() {
922 return delegationToken;
923 }
924
925 @Override
926 public <T extends TokenIdentifier> void setDelegationToken(
927 final Token<T> token) {
928 synchronized(this) {
929 delegationToken = token;
930 }
931 }
932
933 private synchronized long renewDelegationToken(final Token<?> token
934 ) throws IOException {
935 final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
936 TokenArgumentParam dtargParam = new TokenArgumentParam(
937 token.encodeToUrlString());
938 final Map<?, ?> m = run(op, null, dtargParam);
939 return (Long) m.get("long");
940 }
941
942 private synchronized void cancelDelegationToken(final Token<?> token
943 ) throws IOException {
944 final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
945 TokenArgumentParam dtargParam = new TokenArgumentParam(
946 token.encodeToUrlString());
947 run(op, null, dtargParam);
948 }
949
950 @Override
951 public BlockLocation[] getFileBlockLocations(final FileStatus status,
952 final long offset, final long length) throws IOException {
953 if (status == null) {
954 return null;
955 }
956 return getFileBlockLocations(status.getPath(), offset, length);
957 }
958
959 @Override
960 public BlockLocation[] getFileBlockLocations(final Path p,
961 final long offset, final long length) throws IOException {
962 statistics.incrementReadOps(1);
963
964 final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS;
965 final Map<?, ?> m = run(op, p, new OffsetParam(offset),
966 new LengthParam(length));
967 return DFSUtil.locatedBlocks2Locations(JsonUtil.toLocatedBlocks(m));
968 }
969
970 @Override
971 public ContentSummary getContentSummary(final Path p) throws IOException {
972 statistics.incrementReadOps(1);
973
974 final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY;
975 final Map<?, ?> m = run(op, p);
976 return JsonUtil.toContentSummary(m);
977 }
978
979 @Override
980 public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
981 ) throws IOException {
982 statistics.incrementReadOps(1);
983
984 final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
985 final Map<?, ?> m = run(op, p);
986 return JsonUtil.toMD5MD5CRC32FileChecksum(m);
987 }
988
989 /** Delegation token renewer. */
990 public static class DtRenewer extends TokenRenewer {
991 @Override
992 public boolean handleKind(Text kind) {
993 return kind.equals(TOKEN_KIND);
994 }
995
996 @Override
997 public boolean isManaged(Token<?> token) throws IOException {
998 return true;
999 }
1000
1001 private static WebHdfsFileSystem getWebHdfs(
1002 final Token<?> token, final Configuration conf) throws IOException {
1003
1004 final InetSocketAddress nnAddr = SecurityUtil.getTokenServiceAddr(token);
1005 final URI uri = DFSUtil.createUri(WebHdfsFileSystem.SCHEME, nnAddr);
1006 return (WebHdfsFileSystem)FileSystem.get(uri, conf);
1007 }
1008
1009 @Override
1010 public long renew(final Token<?> token, final Configuration conf
1011 ) throws IOException, InterruptedException {
1012 return getWebHdfs(token, conf).renewDelegationToken(token);
1013 }
1014
1015 @Override
1016 public void cancel(final Token<?> token, final Configuration conf
1017 ) throws IOException, InterruptedException {
1018 getWebHdfs(token, conf).cancelDelegationToken(token);
1019 }
1020 }
1021
1022 private static class WebHdfsDelegationTokenSelector
1023 extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
1024 private static final DelegationTokenSelector hdfsTokenSelector =
1025 new DelegationTokenSelector();
1026
1027 public WebHdfsDelegationTokenSelector() {
1028 super(TOKEN_KIND);
1029 }
1030
1031 Token<DelegationTokenIdentifier> selectToken(URI nnUri,
1032 Collection<Token<?>> tokens, Configuration conf) {
1033 Token<DelegationTokenIdentifier> token =
1034 selectToken(SecurityUtil.buildTokenService(nnUri), tokens);
1035 if (token == null) {
1036 token = hdfsTokenSelector.selectToken(nnUri, tokens, conf);
1037 }
1038 return token;
1039 }
1040 }
1041 }