Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.datanode.web.resources;
019
020import java.io.IOException;
021import java.io.InputStream;
022import java.net.URI;
023import java.net.URISyntaxException;
024import java.security.PrivilegedExceptionAction;
025import java.util.EnumSet;
026
027import javax.servlet.ServletContext;
028import javax.servlet.http.HttpServletRequest;
029import javax.servlet.http.HttpServletResponse;
030import javax.ws.rs.Consumes;
031import javax.ws.rs.DefaultValue;
032import javax.ws.rs.GET;
033import javax.ws.rs.POST;
034import javax.ws.rs.PUT;
035import javax.ws.rs.Path;
036import javax.ws.rs.PathParam;
037import javax.ws.rs.Produces;
038import javax.ws.rs.QueryParam;
039import javax.ws.rs.core.Context;
040import javax.ws.rs.core.MediaType;
041import javax.ws.rs.core.Response;
042
043import com.google.common.annotations.VisibleForTesting;
044import org.apache.commons.logging.Log;
045import org.apache.commons.logging.LogFactory;
046import org.apache.hadoop.conf.Configuration;
047import org.apache.hadoop.fs.CreateFlag;
048import org.apache.hadoop.fs.FSDataOutputStream;
049import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
050import org.apache.hadoop.fs.permission.FsPermission;
051import org.apache.hadoop.hdfs.DFSClient;
052import org.apache.hadoop.hdfs.HAUtil;
053import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
054import org.apache.hadoop.hdfs.protocol.HdfsConstants;
055import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
056import org.apache.hadoop.hdfs.server.datanode.DataNode;
057import org.apache.hadoop.hdfs.web.JsonUtil;
058import org.apache.hadoop.hdfs.web.ParamFilter;
059import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
060import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
061import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
062import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
063import org.apache.hadoop.hdfs.web.resources.DelegationParam;
064import org.apache.hadoop.hdfs.web.resources.GetOpParam;
065import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
066import org.apache.hadoop.hdfs.web.resources.LengthParam;
067import org.apache.hadoop.hdfs.web.resources.NamenodeAddressParam;
068import org.apache.hadoop.hdfs.web.resources.OffsetParam;
069import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
070import org.apache.hadoop.hdfs.web.resources.Param;
071import org.apache.hadoop.hdfs.web.resources.PermissionParam;
072import org.apache.hadoop.hdfs.web.resources.PostOpParam;
073import org.apache.hadoop.hdfs.web.resources.PutOpParam;
074import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
075import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
076import org.apache.hadoop.io.IOUtils;
077import org.apache.hadoop.io.Text;
078import org.apache.hadoop.security.SecurityUtil;
079import org.apache.hadoop.security.UserGroupInformation;
080import org.apache.hadoop.security.token.Token;
081
082import com.sun.jersey.spi.container.ResourceFilters;
083
084/** Web-hdfs DataNode implementation. */
085@Path("")
086@ResourceFilters(ParamFilter.class)
087public class DatanodeWebHdfsMethods {
088  public static final Log LOG = LogFactory.getLog(DatanodeWebHdfsMethods.class);
089
090  private static final UriFsPathParam ROOT = new UriFsPathParam("");
091
092  private @Context ServletContext context;
093  private @Context HttpServletRequest request;
094  private @Context HttpServletResponse response;
095
096  private void init(final UserGroupInformation ugi,
097      final DelegationParam delegation, final String nnId,
098      final UriFsPathParam path, final HttpOpParam<?> op,
099      final Param<?, ?>... parameters) throws IOException {
100    if (LOG.isTraceEnabled()) {
101      LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path
102          + ", ugi=" + ugi + Param.toSortedString(", ", parameters));
103    }
104    if (nnId == null) {
105      throw new IllegalArgumentException(NamenodeAddressParam.NAME
106          + " is not specified.");
107    }
108
109    //clear content type
110    response.setContentType(null);
111    
112    if (UserGroupInformation.isSecurityEnabled()) {
113      //add a token for RPC.
114      final Token<DelegationTokenIdentifier> token = deserializeToken
115              (delegation.getValue(), nnId);
116      ugi.addToken(token);
117    }
118  }
119
120  @VisibleForTesting
121  Token<DelegationTokenIdentifier> deserializeToken
122          (String delegation,String nnId) throws IOException {
123    final DataNode datanode = (DataNode) context.getAttribute("datanode");
124    final Configuration conf = datanode.getConf();
125    final Token<DelegationTokenIdentifier> token = new
126            Token<DelegationTokenIdentifier>();
127    token.decodeFromUrlString(delegation);
128    URI nnUri = URI.create(HdfsConstants.HDFS_URI_SCHEME +
129            "://" + nnId);
130    boolean isHA = HAUtil.isLogicalUri(conf, nnUri);
131    if (isHA) {
132      token.setService(HAUtil.buildTokenServiceForLogicalUri(nnUri));
133    } else {
134      token.setService(SecurityUtil.buildTokenService(nnUri));
135    }
136    token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
137    return token;
138  }
139
140  /** Handle HTTP PUT request for the root. */
141  @PUT
142  @Path("/")
143  @Consumes({"*/*"})
144  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
145  public Response putRoot(
146      final InputStream in,
147      @Context final UserGroupInformation ugi,
148      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
149          final DelegationParam delegation,
150      @QueryParam(NamenodeAddressParam.NAME)
151      @DefaultValue(NamenodeAddressParam.DEFAULT)
152          final NamenodeAddressParam namenode,
153      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
154          final PutOpParam op,
155      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
156          final PermissionParam permission,
157      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
158          final OverwriteParam overwrite,
159      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
160          final BufferSizeParam bufferSize,
161      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
162          final ReplicationParam replication,
163      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
164          final BlockSizeParam blockSize
165      ) throws IOException, InterruptedException {
166    return put(in, ugi, delegation, namenode, ROOT, op, permission,
167            overwrite, bufferSize, replication, blockSize);
168  }
169
170  /** Handle HTTP PUT request. */
171  @PUT
172  @Path("{" + UriFsPathParam.NAME + ":.*}")
173  @Consumes({"*/*"})
174  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
175  public Response put(
176      final InputStream in,
177      @Context final UserGroupInformation ugi,
178      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
179          final DelegationParam delegation,
180      @QueryParam(NamenodeAddressParam.NAME)
181      @DefaultValue(NamenodeAddressParam.DEFAULT)
182          final NamenodeAddressParam namenode,
183      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
184      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
185          final PutOpParam op,
186      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
187          final PermissionParam permission,
188      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
189          final OverwriteParam overwrite,
190      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
191          final BufferSizeParam bufferSize,
192      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
193          final ReplicationParam replication,
194      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
195          final BlockSizeParam blockSize
196      ) throws IOException, InterruptedException {
197
198    final String nnId = namenode.getValue();
199    init(ugi, delegation, nnId, path, op, permission,
200        overwrite, bufferSize, replication, blockSize);
201
202    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
203      @Override
204      public Response run() throws IOException, URISyntaxException {
205        return put(in, nnId, path.getAbsolutePath(), op,
206                permission, overwrite, bufferSize, replication, blockSize);
207      }
208    });
209  }
210
211  private Response put(
212      final InputStream in,
213      final String nnId,
214      final String fullpath,
215      final PutOpParam op,
216      final PermissionParam permission,
217      final OverwriteParam overwrite,
218      final BufferSizeParam bufferSize,
219      final ReplicationParam replication,
220      final BlockSizeParam blockSize
221      ) throws IOException, URISyntaxException {
222    final DataNode datanode = (DataNode)context.getAttribute("datanode");
223
224    switch(op.getValue()) {
225    case CREATE:
226    {
227      final Configuration conf = new Configuration(datanode.getConf());
228      conf.set(FsPermission.UMASK_LABEL, "000");
229
230      final int b = bufferSize.getValue(conf);
231      DFSClient dfsclient = newDfsClient(nnId, conf);
232      FSDataOutputStream out = null;
233      try {
234        out = new FSDataOutputStream(dfsclient.create(
235            fullpath, permission.getFsPermission(), 
236            overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
237                : EnumSet.of(CreateFlag.CREATE),
238            replication.getValue(conf), blockSize.getValue(conf), null, b, null), null);
239        IOUtils.copyBytes(in, out, b);
240        out.close();
241        out = null;
242        dfsclient.close();
243        dfsclient = null;
244      } finally {
245        IOUtils.cleanup(LOG, out);
246        IOUtils.cleanup(LOG, dfsclient);
247      }
248      final String scheme = "http".equals(request.getScheme()) ?
249      WebHdfsFileSystem.SCHEME : SWebHdfsFileSystem.SCHEME;
250      final URI uri = new URI(scheme, nnId, fullpath, null, null);
251      return Response.created(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
252    }
253    default:
254      throw new UnsupportedOperationException(op + " is not supported");
255    }
256  }
257
258  /** Handle HTTP POST request for the root for the root. */
259  @POST
260  @Path("/")
261  @Consumes({"*/*"})
262  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
263  public Response postRoot(
264      final InputStream in,
265      @Context final UserGroupInformation ugi,
266      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
267          final DelegationParam delegation,
268      @QueryParam(NamenodeAddressParam.NAME)
269      @DefaultValue(NamenodeAddressParam.DEFAULT)
270          final NamenodeAddressParam namenode,
271      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
272          final PostOpParam op,
273      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
274          final BufferSizeParam bufferSize
275      ) throws IOException, InterruptedException {
276    return post(in, ugi, delegation, namenode, ROOT, op, bufferSize);
277  }
278
279  /** Handle HTTP POST request. */
280  @POST
281  @Path("{" + UriFsPathParam.NAME + ":.*}")
282  @Consumes({"*/*"})
283  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
284  public Response post(
285      final InputStream in,
286      @Context final UserGroupInformation ugi,
287      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
288          final DelegationParam delegation,
289      @QueryParam(NamenodeAddressParam.NAME)
290      @DefaultValue(NamenodeAddressParam.DEFAULT)
291          final NamenodeAddressParam namenode,
292      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
293      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
294          final PostOpParam op,
295      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
296          final BufferSizeParam bufferSize
297      ) throws IOException, InterruptedException {
298
299    final String nnId = namenode.getValue();
300    init(ugi, delegation, nnId, path, op, bufferSize);
301
302    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
303      @Override
304      public Response run() throws IOException {
305        return post(in, nnId, path.getAbsolutePath(), op,
306                bufferSize);
307      }
308    });
309  }
310
311  private Response post(
312      final InputStream in,
313      final String nnId,
314      final String fullpath,
315      final PostOpParam op,
316      final BufferSizeParam bufferSize
317      ) throws IOException {
318    final DataNode datanode = (DataNode)context.getAttribute("datanode");
319
320    switch(op.getValue()) {
321    case APPEND:
322    {
323      final Configuration conf = new Configuration(datanode.getConf());
324      final int b = bufferSize.getValue(conf);
325      DFSClient dfsclient = newDfsClient(nnId, conf);
326      FSDataOutputStream out = null;
327      try {
328        out = dfsclient.append(fullpath, b, null, null);
329        IOUtils.copyBytes(in, out, b);
330        out.close();
331        out = null;
332        dfsclient.close();
333        dfsclient = null;
334      } finally {
335        IOUtils.cleanup(LOG, out);
336        IOUtils.cleanup(LOG, dfsclient);
337      }
338      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
339    }
340    default:
341      throw new UnsupportedOperationException(op + " is not supported");
342    }
343  }
344
345  /** Handle HTTP GET request for the root. */
346  @GET
347  @Path("/")
348  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
349  public Response getRoot(
350      @Context final UserGroupInformation ugi,
351      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
352          final DelegationParam delegation,
353      @QueryParam(NamenodeAddressParam.NAME)
354      @DefaultValue(NamenodeAddressParam.DEFAULT)
355          final NamenodeAddressParam namenode,
356      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
357          final GetOpParam op,
358      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
359          final OffsetParam offset,
360      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
361          final LengthParam length,
362      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
363          final BufferSizeParam bufferSize
364      ) throws IOException, InterruptedException {
365    return get(ugi, delegation, namenode, ROOT, op, offset, length,
366        bufferSize);
367  }
368
369  /** Handle HTTP GET request. */
370  @GET
371  @Path("{" + UriFsPathParam.NAME + ":.*}")
372  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
373  public Response get(
374      @Context final UserGroupInformation ugi,
375      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
376          final DelegationParam delegation,
377      @QueryParam(NamenodeAddressParam.NAME)
378      @DefaultValue(NamenodeAddressParam.DEFAULT)
379          final NamenodeAddressParam namenode,
380      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
381      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
382          final GetOpParam op,
383      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
384          final OffsetParam offset,
385      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
386          final LengthParam length,
387      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
388          final BufferSizeParam bufferSize
389      ) throws IOException, InterruptedException {
390
391    final String nnId = namenode.getValue();
392    init(ugi, delegation, nnId, path, op, offset, length, bufferSize);
393
394    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
395      @Override
396      public Response run() throws IOException {
397        return get(nnId, path.getAbsolutePath(), op, offset,
398                length, bufferSize);
399      }
400    });
401  }
402
403  private Response get(
404      final String nnId,
405      final String fullpath,
406      final GetOpParam op,
407      final OffsetParam offset,
408      final LengthParam length,
409      final BufferSizeParam bufferSize
410      ) throws IOException {
411    final DataNode datanode = (DataNode)context.getAttribute("datanode");
412    final Configuration conf = new Configuration(datanode.getConf());
413
414    switch(op.getValue()) {
415    case OPEN:
416    {
417      final int b = bufferSize.getValue(conf);
418      final DFSClient dfsclient = newDfsClient(nnId, conf);
419      HdfsDataInputStream in = null;
420      try {
421        in = new HdfsDataInputStream(dfsclient.open(fullpath, b, true));
422        in.seek(offset.getValue());
423      } catch(IOException ioe) {
424        IOUtils.cleanup(LOG, in);
425        IOUtils.cleanup(LOG, dfsclient);
426        throw ioe;
427      }
428      
429      final long n = length.getValue() != null ?
430        Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
431        in.getVisibleLength() - offset.getValue();
432
433      /**
434       * Allow the Web UI to perform an AJAX request to get the data.
435       */
436      return Response.ok(new OpenEntity(in, n, dfsclient))
437          .type(MediaType.APPLICATION_OCTET_STREAM)
438          .header("Access-Control-Allow-Methods", "GET")
439          .header("Access-Control-Allow-Origin", "*")
440          .build();
441    }
442    case GETFILECHECKSUM:
443    {
444      MD5MD5CRC32FileChecksum checksum = null;
445      DFSClient dfsclient = newDfsClient(nnId, conf);
446      try {
447        checksum = dfsclient.getFileChecksum(fullpath);
448        dfsclient.close();
449        dfsclient = null;
450      } finally {
451        IOUtils.cleanup(LOG, dfsclient);
452      }
453      final String js = JsonUtil.toJsonString(checksum);
454      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
455    }
456    default:
457      throw new UnsupportedOperationException(op + " is not supported");
458    }
459  }
460
461  private static DFSClient newDfsClient(String nnId,
462                                        Configuration conf) throws IOException {
463    URI uri = URI.create(HdfsConstants.HDFS_URI_SCHEME + "://" + nnId);
464    return new DFSClient(uri, conf);
465  }
466}