Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.namenode.web.resources;
019
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import java.io.OutputStream;
023import java.io.OutputStreamWriter;
024import java.io.PrintWriter;
025import java.net.InetAddress;
026import java.net.URI;
027import java.net.URISyntaxException;
028import java.security.PrivilegedExceptionAction;
029import java.util.ArrayList;
030import java.util.EnumSet;
031
032import javax.servlet.ServletContext;
033import javax.servlet.http.HttpServletRequest;
034import javax.servlet.http.HttpServletResponse;
035import javax.ws.rs.Consumes;
036import javax.ws.rs.DELETE;
037import javax.ws.rs.DefaultValue;
038import javax.ws.rs.GET;
039import javax.ws.rs.POST;
040import javax.ws.rs.PUT;
041import javax.ws.rs.Path;
042import javax.ws.rs.PathParam;
043import javax.ws.rs.Produces;
044import javax.ws.rs.QueryParam;
045import javax.ws.rs.core.Context;
046import javax.ws.rs.core.MediaType;
047import javax.ws.rs.core.Response;
048import javax.ws.rs.core.StreamingOutput;
049
050import org.apache.commons.logging.Log;
051import org.apache.commons.logging.LogFactory;
052import org.apache.hadoop.conf.Configuration;
053import org.apache.hadoop.fs.ContentSummary;
054import org.apache.hadoop.fs.FileStatus;
055import org.apache.hadoop.fs.Options;
056import org.apache.hadoop.fs.permission.AclStatus;
057import org.apache.hadoop.hdfs.StorageType;
058import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
059import org.apache.hadoop.hdfs.protocol.DirectoryListing;
060import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
061import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
062import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
063import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
064import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
065import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
066import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
067import org.apache.hadoop.hdfs.server.common.JspHelper;
068import org.apache.hadoop.hdfs.server.namenode.NameNode;
069import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
070import org.apache.hadoop.hdfs.web.JsonUtil;
071import org.apache.hadoop.hdfs.web.ParamFilter;
072import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
073import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
074import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
075import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
076import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
077import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
078import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam;
079import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
080import org.apache.hadoop.hdfs.web.resources.DelegationParam;
081import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
082import org.apache.hadoop.hdfs.web.resources.DestinationParam;
083import org.apache.hadoop.hdfs.web.resources.DoAsParam;
084import org.apache.hadoop.hdfs.web.resources.GetOpParam;
085import org.apache.hadoop.hdfs.web.resources.GroupParam;
086import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
087import org.apache.hadoop.hdfs.web.resources.LengthParam;
088import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
089import org.apache.hadoop.hdfs.web.resources.NamenodeAddressParam;
090import org.apache.hadoop.hdfs.web.resources.OffsetParam;
091import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
092import org.apache.hadoop.hdfs.web.resources.OwnerParam;
093import org.apache.hadoop.hdfs.web.resources.Param;
094import org.apache.hadoop.hdfs.web.resources.PermissionParam;
095import org.apache.hadoop.hdfs.web.resources.PostOpParam;
096import org.apache.hadoop.hdfs.web.resources.PutOpParam;
097import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
098import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
099import org.apache.hadoop.hdfs.web.resources.RenewerParam;
100import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
101import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
102import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
103import org.apache.hadoop.hdfs.web.resources.UserParam;
104import org.apache.hadoop.io.Text;
105import org.apache.hadoop.ipc.Server;
106import org.apache.hadoop.net.NetworkTopology.InvalidTopologyException;
107import org.apache.hadoop.net.NodeBase;
108import org.apache.hadoop.security.Credentials;
109import org.apache.hadoop.security.UserGroupInformation;
110import org.apache.hadoop.security.token.Token;
111import org.apache.hadoop.security.token.TokenIdentifier;
112
113import com.google.common.annotations.VisibleForTesting;
114import com.google.common.base.Charsets;
115import com.sun.jersey.spi.container.ResourceFilters;
116
117/** Web-hdfs NameNode implementation. */
118@Path("")
119@ResourceFilters(ParamFilter.class)
120public class NamenodeWebHdfsMethods {
121  public static final Log LOG = LogFactory.getLog(NamenodeWebHdfsMethods.class);
122
123  private static final UriFsPathParam ROOT = new UriFsPathParam("");
124  
125  private static final ThreadLocal<String> REMOTE_ADDRESS = new ThreadLocal<String>(); 
126
127  /** @return the remote client address. */
128  public static String getRemoteAddress() {
129    return REMOTE_ADDRESS.get();
130  }
131
132  public static InetAddress getRemoteIp() {
133    try {
134      return InetAddress.getByName(getRemoteAddress());
135    } catch (Exception e) {
136      return null;
137    }
138  }
139
140  /**
141   * Returns true if a WebHdfs request is in progress.  Akin to
142   * {@link Server#isRpcInvocation()}.
143   */
144  public static boolean isWebHdfsInvocation() {
145    return getRemoteAddress() != null;
146  }
147
148  private @Context ServletContext context;
149  private @Context HttpServletRequest request;
150  private @Context HttpServletResponse response;
151
152  private void init(final UserGroupInformation ugi,
153      final DelegationParam delegation,
154      final UserParam username, final DoAsParam doAsUser,
155      final UriFsPathParam path, final HttpOpParam<?> op,
156      final Param<?, ?>... parameters) {
157    if (LOG.isTraceEnabled()) {
158      LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path
159          + ", ugi=" + ugi + ", " + username + ", " + doAsUser
160          + Param.toSortedString(", ", parameters));
161    }
162
163    //clear content type
164    response.setContentType(null);
165  }
166
167  private static NamenodeProtocols getRPCServer(NameNode namenode)
168      throws IOException {
169     final NamenodeProtocols np = namenode.getRpcServer();
170     if (np == null) {
171       throw new IOException("Namenode is in startup mode");
172     }
173     return np;
174  }
175
176  @VisibleForTesting
177  static DatanodeInfo chooseDatanode(final NameNode namenode,
178      final String path, final HttpOpParam.Op op, final long openOffset,
179      final long blocksize) throws IOException {
180    final BlockManager bm = namenode.getNamesystem().getBlockManager();
181
182    if (op == PutOpParam.Op.CREATE) {
183      //choose a datanode near to client 
184      final DatanodeDescriptor clientNode = bm.getDatanodeManager(
185          ).getDatanodeByHost(getRemoteAddress());
186      if (clientNode != null) {
187        final DatanodeStorageInfo[] storages = bm.getBlockPlacementPolicy()
188            .chooseTarget(path, 1, clientNode,
189                new ArrayList<DatanodeStorageInfo>(), false, null, blocksize,
190                // TODO: get storage type from the file
191                StorageType.DEFAULT);
192        if (storages.length > 0) {
193          return storages[0].getDatanodeDescriptor();
194        }
195      }
196    } else if (op == GetOpParam.Op.OPEN
197        || op == GetOpParam.Op.GETFILECHECKSUM
198        || op == PostOpParam.Op.APPEND) {
199      //choose a datanode containing a replica 
200      final NamenodeProtocols np = getRPCServer(namenode);
201      final HdfsFileStatus status = np.getFileInfo(path);
202      if (status == null) {
203        throw new FileNotFoundException("File " + path + " not found.");
204      }
205      final long len = status.getLen();
206      if (op == GetOpParam.Op.OPEN) {
207        if (openOffset < 0L || (openOffset >= len && len > 0)) {
208          throw new IOException("Offset=" + openOffset
209              + " out of the range [0, " + len + "); " + op + ", path=" + path);
210        }
211      }
212
213      if (len > 0) {
214        final long offset = op == GetOpParam.Op.OPEN? openOffset: len - 1;
215        final LocatedBlocks locations = np.getBlockLocations(path, offset, 1);
216        final int count = locations.locatedBlockCount();
217        if (count > 0) {
218          return bestNode(locations.get(0).getLocations());
219        }
220      }
221    } 
222
223    return (DatanodeDescriptor)bm.getDatanodeManager().getNetworkTopology(
224        ).chooseRandom(NodeBase.ROOT);
225  }
226
227  /**
228   * Choose the datanode to redirect the request. Note that the nodes have been
229   * sorted based on availability and network distances, thus it is sufficient
230   * to return the first element of the node here.
231   */
232  private static DatanodeInfo bestNode(DatanodeInfo[] nodes) throws IOException {
233    if (nodes.length == 0 || nodes[0].isDecommissioned()) {
234      throw new IOException("No active nodes contain this block");
235    }
236    return nodes[0];
237  }
238
239  private Token<? extends TokenIdentifier> generateDelegationToken(
240      final NameNode namenode, final UserGroupInformation ugi,
241      final String renewer) throws IOException {
242    final Credentials c = DelegationTokenSecretManager.createCredentials(
243        namenode, ugi, renewer != null? renewer: ugi.getShortUserName());
244    final Token<? extends TokenIdentifier> t = c.getAllTokens().iterator().next();
245    Text kind = request.getScheme().equals("http") ? WebHdfsFileSystem.TOKEN_KIND
246        : SWebHdfsFileSystem.TOKEN_KIND;
247    t.setKind(kind);
248    return t;
249  }
250
251  private URI redirectURI(final NameNode namenode,
252      final UserGroupInformation ugi, final DelegationParam delegation,
253      final UserParam username, final DoAsParam doAsUser,
254      final String path, final HttpOpParam.Op op, final long openOffset,
255      final long blocksize,
256      final Param<?, ?>... parameters) throws URISyntaxException, IOException {
257    final DatanodeInfo dn;
258    try {
259      dn = chooseDatanode(namenode, path, op, openOffset, blocksize);
260    } catch (InvalidTopologyException ite) {
261      throw new IOException("Failed to find datanode, suggest to check cluster health.", ite);
262    }
263
264    final String delegationQuery;
265    if (!UserGroupInformation.isSecurityEnabled()) {
266      //security disabled
267      delegationQuery = Param.toSortedString("&", doAsUser, username);
268    } else if (delegation.getValue() != null) {
269      //client has provided a token
270      delegationQuery = "&" + delegation;
271    } else {
272      //generate a token
273      final Token<? extends TokenIdentifier> t = generateDelegationToken(
274          namenode, ugi, request.getUserPrincipal().getName());
275      delegationQuery = "&" + new DelegationParam(t.encodeToUrlString());
276    }
277    final String query = op.toQueryString() + delegationQuery
278        + "&" + new NamenodeAddressParam(namenode)
279        + Param.toSortedString("&", parameters);
280    final String uripath = WebHdfsFileSystem.PATH_PREFIX + path;
281
282    final String scheme = request.getScheme();
283    int port = "http".equals(scheme) ? dn.getInfoPort() : dn
284        .getInfoSecurePort();
285    final URI uri = new URI(scheme, null, dn.getHostName(), port, uripath,
286        query, null);
287
288    if (LOG.isTraceEnabled()) {
289      LOG.trace("redirectURI=" + uri);
290    }
291    return uri;
292  }
293
294  /** Handle HTTP PUT request for the root. */
295  @PUT
296  @Path("/")
297  @Consumes({"*/*"})
298  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
299  public Response putRoot(
300      @Context final UserGroupInformation ugi,
301      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
302          final DelegationParam delegation,
303      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
304          final UserParam username,
305      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
306          final DoAsParam doAsUser,
307      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
308          final PutOpParam op,
309      @QueryParam(DestinationParam.NAME) @DefaultValue(DestinationParam.DEFAULT)
310          final DestinationParam destination,
311      @QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT)
312          final OwnerParam owner,
313      @QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT)
314          final GroupParam group,
315      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
316          final PermissionParam permission,
317      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
318          final OverwriteParam overwrite,
319      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
320          final BufferSizeParam bufferSize,
321      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
322          final ReplicationParam replication,
323      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
324          final BlockSizeParam blockSize,
325      @QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT)
326          final ModificationTimeParam modificationTime,
327      @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT)
328          final AccessTimeParam accessTime,
329      @QueryParam(RenameOptionSetParam.NAME) @DefaultValue(RenameOptionSetParam.DEFAULT)
330          final RenameOptionSetParam renameOptions,
331      @QueryParam(CreateParentParam.NAME) @DefaultValue(CreateParentParam.DEFAULT)
332          final CreateParentParam createParent,
333      @QueryParam(TokenArgumentParam.NAME) @DefaultValue(TokenArgumentParam.DEFAULT)
334          final TokenArgumentParam delegationTokenArgument,
335      @QueryParam(AclPermissionParam.NAME) @DefaultValue(AclPermissionParam.DEFAULT) 
336          final AclPermissionParam aclPermission
337          )throws IOException, InterruptedException {
338    return put(ugi, delegation, username, doAsUser, ROOT, op, destination,
339        owner, group, permission, overwrite, bufferSize, replication,
340        blockSize, modificationTime, accessTime, renameOptions, createParent,
341        delegationTokenArgument,aclPermission);
342  }
343
344  /** Handle HTTP PUT request. */
345  @PUT
346  @Path("{" + UriFsPathParam.NAME + ":.*}")
347  @Consumes({"*/*"})
348  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
349  public Response put(
350      @Context final UserGroupInformation ugi,
351      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
352          final DelegationParam delegation,
353      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
354          final UserParam username,
355      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
356          final DoAsParam doAsUser,
357      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
358      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
359          final PutOpParam op,
360      @QueryParam(DestinationParam.NAME) @DefaultValue(DestinationParam.DEFAULT)
361          final DestinationParam destination,
362      @QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT)
363          final OwnerParam owner,
364      @QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT)
365          final GroupParam group,
366      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
367          final PermissionParam permission,
368      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
369          final OverwriteParam overwrite,
370      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
371          final BufferSizeParam bufferSize,
372      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
373          final ReplicationParam replication,
374      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
375          final BlockSizeParam blockSize,
376      @QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT)
377          final ModificationTimeParam modificationTime,
378      @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT)
379          final AccessTimeParam accessTime,
380      @QueryParam(RenameOptionSetParam.NAME) @DefaultValue(RenameOptionSetParam.DEFAULT)
381          final RenameOptionSetParam renameOptions,
382      @QueryParam(CreateParentParam.NAME) @DefaultValue(CreateParentParam.DEFAULT)
383          final CreateParentParam createParent,
384      @QueryParam(TokenArgumentParam.NAME) @DefaultValue(TokenArgumentParam.DEFAULT)
385          final TokenArgumentParam delegationTokenArgument,
386      @QueryParam(AclPermissionParam.NAME) @DefaultValue(AclPermissionParam.DEFAULT) 
387          final AclPermissionParam aclPermission
388      ) throws IOException, InterruptedException {
389
390    init(ugi, delegation, username, doAsUser, path, op, destination, owner,
391        group, permission, overwrite, bufferSize, replication, blockSize,
392        modificationTime, accessTime, renameOptions, delegationTokenArgument,aclPermission);
393
394    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
395      @Override
396      public Response run() throws IOException, URISyntaxException {
397        REMOTE_ADDRESS.set(request.getRemoteAddr());
398        try {
399          return put(ugi, delegation, username, doAsUser,
400              path.getAbsolutePath(), op, destination, owner, group,
401              permission, overwrite, bufferSize, replication, blockSize,
402              modificationTime, accessTime, renameOptions, createParent,
403              delegationTokenArgument,aclPermission);
404        } finally {
405          REMOTE_ADDRESS.set(null);
406        }
407      }
408    });
409  }
410
411  private Response put(
412      final UserGroupInformation ugi,
413      final DelegationParam delegation,
414      final UserParam username,
415      final DoAsParam doAsUser,
416      final String fullpath,
417      final PutOpParam op,
418      final DestinationParam destination,
419      final OwnerParam owner,
420      final GroupParam group,
421      final PermissionParam permission,
422      final OverwriteParam overwrite,
423      final BufferSizeParam bufferSize,
424      final ReplicationParam replication,
425      final BlockSizeParam blockSize,
426      final ModificationTimeParam modificationTime,
427      final AccessTimeParam accessTime,
428      final RenameOptionSetParam renameOptions,
429      final CreateParentParam createParent,
430      final TokenArgumentParam delegationTokenArgument,
431      final AclPermissionParam aclPermission
432      ) throws IOException, URISyntaxException {
433
434    final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF);
435    final NameNode namenode = (NameNode)context.getAttribute("name.node");
436    final NamenodeProtocols np = getRPCServer(namenode);
437
438    switch(op.getValue()) {
439    case CREATE:
440    {
441      final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
442          fullpath, op.getValue(), -1L, blockSize.getValue(conf),
443          permission, overwrite, bufferSize, replication, blockSize);
444      return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
445    } 
446    case MKDIRS:
447    {
448      final boolean b = np.mkdirs(fullpath, permission.getFsPermission(), true);
449      final String js = JsonUtil.toJsonString("boolean", b);
450      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
451    }
452    case CREATESYMLINK:
453    {
454      np.createSymlink(destination.getValue(), fullpath,
455          PermissionParam.getDefaultFsPermission(), createParent.getValue());
456      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
457    }
458    case RENAME:
459    {
460      final EnumSet<Options.Rename> s = renameOptions.getValue();
461      if (s.isEmpty()) {
462        final boolean b = np.rename(fullpath, destination.getValue());
463        final String js = JsonUtil.toJsonString("boolean", b);
464        return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
465      } else {
466        np.rename2(fullpath, destination.getValue(),
467            s.toArray(new Options.Rename[s.size()]));
468        return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
469      }
470    }
471    case SETREPLICATION:
472    {
473      final boolean b = np.setReplication(fullpath, replication.getValue(conf));
474      final String js = JsonUtil.toJsonString("boolean", b);
475      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
476    }
477    case SETOWNER:
478    {
479      if (owner.getValue() == null && group.getValue() == null) {
480        throw new IllegalArgumentException("Both owner and group are empty.");
481      }
482
483      np.setOwner(fullpath, owner.getValue(), group.getValue());
484      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
485    }
486    case SETPERMISSION:
487    {
488      np.setPermission(fullpath, permission.getFsPermission());
489      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
490    }
491    case SETTIMES:
492    {
493      np.setTimes(fullpath, modificationTime.getValue(), accessTime.getValue());
494      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
495    }
496    case RENEWDELEGATIONTOKEN:
497    {
498      final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
499      token.decodeFromUrlString(delegationTokenArgument.getValue());
500      final long expiryTime = np.renewDelegationToken(token);
501      final String js = JsonUtil.toJsonString("long", expiryTime);
502      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
503    }
504    case CANCELDELEGATIONTOKEN:
505    {
506      final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
507      token.decodeFromUrlString(delegationTokenArgument.getValue());
508      np.cancelDelegationToken(token);
509      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
510    }
511    case MODIFYACLENTRIES: {
512      np.modifyAclEntries(fullpath, aclPermission.getAclPermission(true));
513      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
514    }
515    case REMOVEACLENTRIES: {
516      np.removeAclEntries(fullpath, aclPermission.getAclPermission(false));
517      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
518    }
519    case REMOVEDEFAULTACL: {
520      np.removeDefaultAcl(fullpath);
521      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
522    }
523    case REMOVEACL: {
524      np.removeAcl(fullpath);
525      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
526    }
527    case SETACL: {
528      np.setAcl(fullpath, aclPermission.getAclPermission(true));
529      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
530    }
531    default:
532      throw new UnsupportedOperationException(op + " is not supported");
533    }
534  }
535
536  /** Handle HTTP POST request for the root. */
537  @POST
538  @Path("/")
539  @Consumes({"*/*"})
540  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
541  public Response postRoot(
542      @Context final UserGroupInformation ugi,
543      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
544          final DelegationParam delegation,
545      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
546          final UserParam username,
547      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
548          final DoAsParam doAsUser,
549      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
550          final PostOpParam op,
551      @QueryParam(ConcatSourcesParam.NAME) @DefaultValue(ConcatSourcesParam.DEFAULT)
552          final ConcatSourcesParam concatSrcs,
553      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
554          final BufferSizeParam bufferSize
555      ) throws IOException, InterruptedException {
556    return post(ugi, delegation, username, doAsUser, ROOT, op, concatSrcs, bufferSize);
557  }
558
559  /** Handle HTTP POST request. */
560  @POST
561  @Path("{" + UriFsPathParam.NAME + ":.*}")
562  @Consumes({"*/*"})
563  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
564  public Response post(
565      @Context final UserGroupInformation ugi,
566      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
567          final DelegationParam delegation,
568      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
569          final UserParam username,
570      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
571          final DoAsParam doAsUser,
572      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
573      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
574          final PostOpParam op,
575      @QueryParam(ConcatSourcesParam.NAME) @DefaultValue(ConcatSourcesParam.DEFAULT)
576          final ConcatSourcesParam concatSrcs,
577      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
578          final BufferSizeParam bufferSize
579      ) throws IOException, InterruptedException {
580
581    init(ugi, delegation, username, doAsUser, path, op, concatSrcs, bufferSize);
582
583    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
584      @Override
585      public Response run() throws IOException, URISyntaxException {
586        REMOTE_ADDRESS.set(request.getRemoteAddr());
587        try {
588          return post(ugi, delegation, username, doAsUser,
589              path.getAbsolutePath(), op, concatSrcs, bufferSize);
590        } finally {
591          REMOTE_ADDRESS.set(null);
592        }
593      }
594    });
595  }
596
597  private Response post(
598      final UserGroupInformation ugi,
599      final DelegationParam delegation,
600      final UserParam username,
601      final DoAsParam doAsUser,
602      final String fullpath,
603      final PostOpParam op,
604      final ConcatSourcesParam concatSrcs,
605      final BufferSizeParam bufferSize
606      ) throws IOException, URISyntaxException {
607    final NameNode namenode = (NameNode)context.getAttribute("name.node");
608
609    switch(op.getValue()) {
610    case APPEND:
611    {
612      final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
613          fullpath, op.getValue(), -1L, -1L, bufferSize);
614      return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
615    }
616    case CONCAT:
617    {
618      getRPCServer(namenode).concat(fullpath, concatSrcs.getAbsolutePaths());
619      return Response.ok().build();
620    }
621    default:
622      throw new UnsupportedOperationException(op + " is not supported");
623    }
624  }
625
626  /** Handle HTTP GET request for the root. */
627  @GET
628  @Path("/")
629  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
630  public Response getRoot(
631      @Context final UserGroupInformation ugi,
632      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
633          final DelegationParam delegation,
634      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
635          final UserParam username,
636      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
637          final DoAsParam doAsUser,
638      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
639          final GetOpParam op,
640      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
641          final OffsetParam offset,
642      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
643          final LengthParam length,
644      @QueryParam(RenewerParam.NAME) @DefaultValue(RenewerParam.DEFAULT)
645          final RenewerParam renewer,
646      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
647          final BufferSizeParam bufferSize
648      ) throws IOException, InterruptedException {
649    return get(ugi, delegation, username, doAsUser, ROOT, op,
650        offset, length, renewer, bufferSize);
651  }
652
653  /** Handle HTTP GET request. */
654  @GET
655  @Path("{" + UriFsPathParam.NAME + ":.*}")
656  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
657  public Response get(
658      @Context final UserGroupInformation ugi,
659      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
660          final DelegationParam delegation,
661      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
662          final UserParam username,
663      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
664          final DoAsParam doAsUser,
665      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
666      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
667          final GetOpParam op,
668      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
669          final OffsetParam offset,
670      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
671          final LengthParam length,
672      @QueryParam(RenewerParam.NAME) @DefaultValue(RenewerParam.DEFAULT)
673          final RenewerParam renewer,
674      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
675          final BufferSizeParam bufferSize
676      ) throws IOException, InterruptedException {
677
678    init(ugi, delegation, username, doAsUser, path, op,
679        offset, length, renewer, bufferSize);
680
681    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
682      @Override
683      public Response run() throws IOException, URISyntaxException {
684        REMOTE_ADDRESS.set(request.getRemoteAddr());
685        try {
686          return get(ugi, delegation, username, doAsUser,
687              path.getAbsolutePath(), op, offset, length, renewer, bufferSize);
688        } finally {
689          REMOTE_ADDRESS.set(null);
690        }
691      }
692    });
693  }
694
695  private Response get(
696      final UserGroupInformation ugi,
697      final DelegationParam delegation,
698      final UserParam username,
699      final DoAsParam doAsUser,
700      final String fullpath,
701      final GetOpParam op,
702      final OffsetParam offset,
703      final LengthParam length,
704      final RenewerParam renewer,
705      final BufferSizeParam bufferSize
706      ) throws IOException, URISyntaxException {
707    final NameNode namenode = (NameNode)context.getAttribute("name.node");
708    final NamenodeProtocols np = getRPCServer(namenode);
709
710    switch(op.getValue()) {
711    case OPEN:
712    {
713      final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
714          fullpath, op.getValue(), offset.getValue(), -1L, offset, length, bufferSize);
715      return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
716    }
717    case GET_BLOCK_LOCATIONS:
718    {
719      final long offsetValue = offset.getValue();
720      final Long lengthValue = length.getValue();
721      final LocatedBlocks locatedblocks = np.getBlockLocations(fullpath,
722          offsetValue, lengthValue != null? lengthValue: Long.MAX_VALUE);
723      final String js = JsonUtil.toJsonString(locatedblocks);
724      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
725    }
726    case GETFILESTATUS:
727    {
728      final HdfsFileStatus status = np.getFileInfo(fullpath);
729      if (status == null) {
730        throw new FileNotFoundException("File does not exist: " + fullpath);
731      }
732
733      final String js = JsonUtil.toJsonString(status, true);
734      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
735    }
736    case LISTSTATUS:
737    {
738      final StreamingOutput streaming = getListingStream(np, fullpath);
739      return Response.ok(streaming).type(MediaType.APPLICATION_JSON).build();
740    }
741    case GETCONTENTSUMMARY:
742    {
743      final ContentSummary contentsummary = np.getContentSummary(fullpath);
744      final String js = JsonUtil.toJsonString(contentsummary);
745      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
746    }
747    case GETFILECHECKSUM:
748    {
749      final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
750          fullpath, op.getValue(), -1L, -1L);
751      return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
752    }
753    case GETDELEGATIONTOKEN:
754    {
755      if (delegation.getValue() != null) {
756        throw new IllegalArgumentException(delegation.getName()
757            + " parameter is not null.");
758      }
759      final Token<? extends TokenIdentifier> token = generateDelegationToken(
760          namenode, ugi, renewer.getValue());
761      final String js = JsonUtil.toJsonString(token);
762      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
763    }
764    case GETHOMEDIRECTORY:
765    {
766      final String js = JsonUtil.toJsonString(
767          org.apache.hadoop.fs.Path.class.getSimpleName(),
768          WebHdfsFileSystem.getHomeDirectoryString(ugi));
769      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
770    }
771    case GETACLSTATUS: {
772      AclStatus status = np.getAclStatus(fullpath);
773      if (status == null) {
774        throw new FileNotFoundException("File does not exist: " + fullpath);
775      }
776
777      final String js = JsonUtil.toJsonString(status);
778      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
779    }
780    default:
781      throw new UnsupportedOperationException(op + " is not supported");
782    }
783  }
784
785  private static DirectoryListing getDirectoryListing(final NamenodeProtocols np,
786      final String p, byte[] startAfter) throws IOException {
787    final DirectoryListing listing = np.getListing(p, startAfter, false);
788    if (listing == null) { // the directory does not exist
789      throw new FileNotFoundException("File " + p + " does not exist.");
790    }
791    return listing;
792  }
793  
794  private static StreamingOutput getListingStream(final NamenodeProtocols np, 
795      final String p) throws IOException {
796    // allows exceptions like FNF or ACE to prevent http response of 200 for
797    // a failure since we can't (currently) return error responses in the
798    // middle of a streaming operation
799    final DirectoryListing firstDirList = getDirectoryListing(np, p,
800        HdfsFileStatus.EMPTY_NAME);
801
802    // must save ugi because the streaming object will be executed outside
803    // the remote user's ugi
804    final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
805    return new StreamingOutput() {
806      @Override
807      public void write(final OutputStream outstream) throws IOException {
808        final PrintWriter out = new PrintWriter(new OutputStreamWriter(
809            outstream, Charsets.UTF_8));
810        out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\""
811            + FileStatus.class.getSimpleName() + "\":[");
812
813        try {
814          // restore remote user's ugi
815          ugi.doAs(new PrivilegedExceptionAction<Void>() {
816            @Override
817            public Void run() throws IOException {
818              long n = 0;
819              for (DirectoryListing dirList = firstDirList; ;
820                   dirList = getDirectoryListing(np, p, dirList.getLastName())
821              ) {
822                // send each segment of the directory listing
823                for (HdfsFileStatus s : dirList.getPartialListing()) {
824                  if (n++ > 0) {
825                    out.println(',');
826                  }
827                  out.print(JsonUtil.toJsonString(s, false));
828                }
829                // stop if last segment
830                if (!dirList.hasMore()) {
831                  break;
832                }
833              }
834              return null;
835            }
836          });
837        } catch (InterruptedException e) {
838          throw new IOException(e);
839        }
840        
841        out.println();
842        out.println("]}}");
843        out.flush();
844      }
845    };
846  }
847
848  /** Handle HTTP DELETE request for the root. */
849  @DELETE
850  @Path("/")
851  @Produces(MediaType.APPLICATION_JSON)
852  public Response deleteRoot(
853      @Context final UserGroupInformation ugi,
854      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
855          final DelegationParam delegation,
856      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
857          final UserParam username,
858      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
859          final DoAsParam doAsUser,
860      @QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT)
861          final DeleteOpParam op,
862      @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT)
863          final RecursiveParam recursive
864      ) throws IOException, InterruptedException {
865    return delete(ugi, delegation, username, doAsUser, ROOT, op, recursive);
866  }
867
868  /** Handle HTTP DELETE request. */
869  @DELETE
870  @Path("{" + UriFsPathParam.NAME + ":.*}")
871  @Produces(MediaType.APPLICATION_JSON)
872  public Response delete(
873      @Context final UserGroupInformation ugi,
874      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
875          final DelegationParam delegation,
876      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
877          final UserParam username,
878      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
879          final DoAsParam doAsUser,
880      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
881      @QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT)
882          final DeleteOpParam op,
883      @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT)
884          final RecursiveParam recursive
885      ) throws IOException, InterruptedException {
886
887    init(ugi, delegation, username, doAsUser, path, op, recursive);
888
889    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
890      @Override
891      public Response run() throws IOException {
892        REMOTE_ADDRESS.set(request.getRemoteAddr());
893        try {
894          return delete(ugi, delegation, username, doAsUser,
895              path.getAbsolutePath(), op, recursive);
896        } finally {
897          REMOTE_ADDRESS.set(null);
898        }
899      }
900    });
901  }
902
903  private Response delete(
904      final UserGroupInformation ugi,
905      final DelegationParam delegation,
906      final UserParam username,
907      final DoAsParam doAsUser,
908      final String fullpath,
909      final DeleteOpParam op,
910      final RecursiveParam recursive
911      ) throws IOException {
912    final NameNode namenode = (NameNode)context.getAttribute("name.node");
913
914    switch(op.getValue()) {
915    case DELETE:
916    {
917      final boolean b = getRPCServer(namenode).delete(fullpath, recursive.getValue());
918      final String js = JsonUtil.toJsonString("boolean", b);
919      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
920    }
921    default:
922      throw new UnsupportedOperationException(op + " is not supported");
923    }
924  }
925}