Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.datanode.web.resources;
019
020import java.io.IOException;
021import java.io.InputStream;
022import java.net.URI;
023import java.net.URISyntaxException;
024import java.security.PrivilegedExceptionAction;
025import java.util.EnumSet;
026
027import javax.servlet.ServletContext;
028import javax.servlet.http.HttpServletRequest;
029import javax.servlet.http.HttpServletResponse;
030import javax.ws.rs.Consumes;
031import javax.ws.rs.DefaultValue;
032import javax.ws.rs.GET;
033import javax.ws.rs.POST;
034import javax.ws.rs.PUT;
035import javax.ws.rs.Path;
036import javax.ws.rs.PathParam;
037import javax.ws.rs.Produces;
038import javax.ws.rs.QueryParam;
039import javax.ws.rs.core.Context;
040import javax.ws.rs.core.MediaType;
041import javax.ws.rs.core.Response;
042
043import com.google.common.annotations.VisibleForTesting;
044import org.apache.commons.logging.Log;
045import org.apache.commons.logging.LogFactory;
046import org.apache.hadoop.conf.Configuration;
047import org.apache.hadoop.fs.CreateFlag;
048import org.apache.hadoop.fs.FSDataOutputStream;
049import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
050import org.apache.hadoop.fs.permission.FsPermission;
051import org.apache.hadoop.hdfs.DFSClient;
052import org.apache.hadoop.hdfs.HAUtil;
053import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
054import org.apache.hadoop.hdfs.protocol.HdfsConstants;
055import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
056import org.apache.hadoop.hdfs.server.datanode.DataNode;
057import org.apache.hadoop.hdfs.web.JsonUtil;
058import org.apache.hadoop.hdfs.web.ParamFilter;
059import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
060import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
061import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
062import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
063import org.apache.hadoop.hdfs.web.resources.DelegationParam;
064import org.apache.hadoop.hdfs.web.resources.GetOpParam;
065import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
066import org.apache.hadoop.hdfs.web.resources.LengthParam;
067import org.apache.hadoop.hdfs.web.resources.NamenodeAddressParam;
068import org.apache.hadoop.hdfs.web.resources.OffsetParam;
069import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
070import org.apache.hadoop.hdfs.web.resources.Param;
071import org.apache.hadoop.hdfs.web.resources.PermissionParam;
072import org.apache.hadoop.hdfs.web.resources.PostOpParam;
073import org.apache.hadoop.hdfs.web.resources.PutOpParam;
074import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
075import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
076import org.apache.hadoop.io.IOUtils;
077import org.apache.hadoop.security.SecurityUtil;
078import org.apache.hadoop.security.UserGroupInformation;
079import org.apache.hadoop.security.token.Token;
080
081import com.sun.jersey.spi.container.ResourceFilters;
082
083/** Web-hdfs DataNode implementation. */
084@Path("")
085@ResourceFilters(ParamFilter.class)
086public class DatanodeWebHdfsMethods {
087  public static final Log LOG = LogFactory.getLog(DatanodeWebHdfsMethods.class);
088
089  private static final UriFsPathParam ROOT = new UriFsPathParam("");
090
091  private @Context ServletContext context;
092  private @Context HttpServletRequest request;
093  private @Context HttpServletResponse response;
094
095  private void init(final UserGroupInformation ugi,
096      final DelegationParam delegation, final String nnId,
097      final UriFsPathParam path, final HttpOpParam<?> op,
098      final Param<?, ?>... parameters) throws IOException {
099    if (LOG.isTraceEnabled()) {
100      LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path
101          + ", ugi=" + ugi + Param.toSortedString(", ", parameters));
102    }
103    if (nnId == null) {
104      throw new IllegalArgumentException(NamenodeAddressParam.NAME
105          + " is not specified.");
106    }
107
108    //clear content type
109    response.setContentType(null);
110    
111    if (UserGroupInformation.isSecurityEnabled()) {
112      //add a token for RPC.
113      final Token<DelegationTokenIdentifier> token = deserializeToken
114              (delegation.getValue(), nnId);
115      ugi.addToken(token);
116    }
117  }
118
119  @VisibleForTesting
120  Token<DelegationTokenIdentifier> deserializeToken
121          (String delegation,String nnId) throws IOException {
122    final DataNode datanode = (DataNode) context.getAttribute("datanode");
123    final Configuration conf = datanode.getConf();
124    final Token<DelegationTokenIdentifier> token = new
125            Token<DelegationTokenIdentifier>();
126    token.decodeFromUrlString(delegation);
127    URI nnUri = URI.create(HdfsConstants.HDFS_URI_SCHEME +
128            "://" + nnId);
129    boolean isLogical = HAUtil.isLogicalUri(conf, nnUri);
130    if (isLogical) {
131      token.setService(HAUtil.buildTokenServiceForLogicalUri(nnUri));
132    } else {
133      token.setService(SecurityUtil.buildTokenService(nnUri));
134    }
135    token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
136    return token;
137  }
138
139  /** Handle HTTP PUT request for the root. */
140  @PUT
141  @Path("/")
142  @Consumes({"*/*"})
143  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
144  public Response putRoot(
145      final InputStream in,
146      @Context final UserGroupInformation ugi,
147      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
148          final DelegationParam delegation,
149      @QueryParam(NamenodeAddressParam.NAME)
150      @DefaultValue(NamenodeAddressParam.DEFAULT)
151          final NamenodeAddressParam namenode,
152      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
153          final PutOpParam op,
154      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
155          final PermissionParam permission,
156      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
157          final OverwriteParam overwrite,
158      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
159          final BufferSizeParam bufferSize,
160      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
161          final ReplicationParam replication,
162      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
163          final BlockSizeParam blockSize
164      ) throws IOException, InterruptedException {
165    return put(in, ugi, delegation, namenode, ROOT, op, permission,
166            overwrite, bufferSize, replication, blockSize);
167  }
168
169  /** Handle HTTP PUT request. */
170  @PUT
171  @Path("{" + UriFsPathParam.NAME + ":.*}")
172  @Consumes({"*/*"})
173  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
174  public Response put(
175      final InputStream in,
176      @Context final UserGroupInformation ugi,
177      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
178          final DelegationParam delegation,
179      @QueryParam(NamenodeAddressParam.NAME)
180      @DefaultValue(NamenodeAddressParam.DEFAULT)
181          final NamenodeAddressParam namenode,
182      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
183      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
184          final PutOpParam op,
185      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
186          final PermissionParam permission,
187      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
188          final OverwriteParam overwrite,
189      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
190          final BufferSizeParam bufferSize,
191      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
192          final ReplicationParam replication,
193      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
194          final BlockSizeParam blockSize
195      ) throws IOException, InterruptedException {
196
197    final String nnId = namenode.getValue();
198    init(ugi, delegation, nnId, path, op, permission,
199        overwrite, bufferSize, replication, blockSize);
200
201    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
202      @Override
203      public Response run() throws IOException, URISyntaxException {
204        return put(in, nnId, path.getAbsolutePath(), op,
205                permission, overwrite, bufferSize, replication, blockSize);
206      }
207    });
208  }
209
210  private Response put(
211      final InputStream in,
212      final String nnId,
213      final String fullpath,
214      final PutOpParam op,
215      final PermissionParam permission,
216      final OverwriteParam overwrite,
217      final BufferSizeParam bufferSize,
218      final ReplicationParam replication,
219      final BlockSizeParam blockSize
220      ) throws IOException, URISyntaxException {
221    final DataNode datanode = (DataNode)context.getAttribute("datanode");
222
223    switch(op.getValue()) {
224    case CREATE:
225    {
226      final Configuration conf = new Configuration(datanode.getConf());
227      conf.set(FsPermission.UMASK_LABEL, "000");
228
229      final int b = bufferSize.getValue(conf);
230      DFSClient dfsclient = newDfsClient(nnId, conf);
231      FSDataOutputStream out = null;
232      try {
233        out = new FSDataOutputStream(dfsclient.create(
234            fullpath, permission.getFsPermission(), 
235            overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
236                : EnumSet.of(CreateFlag.CREATE),
237            replication.getValue(conf), blockSize.getValue(conf), null, b, null), null);
238        IOUtils.copyBytes(in, out, b);
239        out.close();
240        out = null;
241        dfsclient.close();
242        dfsclient = null;
243      } finally {
244        IOUtils.cleanup(LOG, out);
245        IOUtils.cleanup(LOG, dfsclient);
246      }
247      final String scheme = "http".equals(request.getScheme()) ?
248      WebHdfsFileSystem.SCHEME : SWebHdfsFileSystem.SCHEME;
249      final URI uri = new URI(scheme, nnId, fullpath, null, null);
250      return Response.created(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
251    }
252    default:
253      throw new UnsupportedOperationException(op + " is not supported");
254    }
255  }
256
257  /** Handle HTTP POST request for the root for the root. */
258  @POST
259  @Path("/")
260  @Consumes({"*/*"})
261  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
262  public Response postRoot(
263      final InputStream in,
264      @Context final UserGroupInformation ugi,
265      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
266          final DelegationParam delegation,
267      @QueryParam(NamenodeAddressParam.NAME)
268      @DefaultValue(NamenodeAddressParam.DEFAULT)
269          final NamenodeAddressParam namenode,
270      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
271          final PostOpParam op,
272      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
273          final BufferSizeParam bufferSize
274      ) throws IOException, InterruptedException {
275    return post(in, ugi, delegation, namenode, ROOT, op, bufferSize);
276  }
277
278  /** Handle HTTP POST request. */
279  @POST
280  @Path("{" + UriFsPathParam.NAME + ":.*}")
281  @Consumes({"*/*"})
282  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
283  public Response post(
284      final InputStream in,
285      @Context final UserGroupInformation ugi,
286      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
287          final DelegationParam delegation,
288      @QueryParam(NamenodeAddressParam.NAME)
289      @DefaultValue(NamenodeAddressParam.DEFAULT)
290          final NamenodeAddressParam namenode,
291      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
292      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
293          final PostOpParam op,
294      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
295          final BufferSizeParam bufferSize
296      ) throws IOException, InterruptedException {
297
298    final String nnId = namenode.getValue();
299    init(ugi, delegation, nnId, path, op, bufferSize);
300
301    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
302      @Override
303      public Response run() throws IOException {
304        return post(in, nnId, path.getAbsolutePath(), op,
305                bufferSize);
306      }
307    });
308  }
309
310  private Response post(
311      final InputStream in,
312      final String nnId,
313      final String fullpath,
314      final PostOpParam op,
315      final BufferSizeParam bufferSize
316      ) throws IOException {
317    final DataNode datanode = (DataNode)context.getAttribute("datanode");
318
319    switch(op.getValue()) {
320    case APPEND:
321    {
322      final Configuration conf = new Configuration(datanode.getConf());
323      final int b = bufferSize.getValue(conf);
324      DFSClient dfsclient = newDfsClient(nnId, conf);
325      FSDataOutputStream out = null;
326      try {
327        out = dfsclient.append(fullpath, b, null, null);
328        IOUtils.copyBytes(in, out, b);
329        out.close();
330        out = null;
331        dfsclient.close();
332        dfsclient = null;
333      } finally {
334        IOUtils.cleanup(LOG, out);
335        IOUtils.cleanup(LOG, dfsclient);
336      }
337      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
338    }
339    default:
340      throw new UnsupportedOperationException(op + " is not supported");
341    }
342  }
343
344  /** Handle HTTP GET request for the root. */
345  @GET
346  @Path("/")
347  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
348  public Response getRoot(
349      @Context final UserGroupInformation ugi,
350      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
351          final DelegationParam delegation,
352      @QueryParam(NamenodeAddressParam.NAME)
353      @DefaultValue(NamenodeAddressParam.DEFAULT)
354          final NamenodeAddressParam namenode,
355      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
356          final GetOpParam op,
357      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
358          final OffsetParam offset,
359      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
360          final LengthParam length,
361      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
362          final BufferSizeParam bufferSize
363      ) throws IOException, InterruptedException {
364    return get(ugi, delegation, namenode, ROOT, op, offset, length,
365        bufferSize);
366  }
367
368  /** Handle HTTP GET request. */
369  @GET
370  @Path("{" + UriFsPathParam.NAME + ":.*}")
371  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
372  public Response get(
373      @Context final UserGroupInformation ugi,
374      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
375          final DelegationParam delegation,
376      @QueryParam(NamenodeAddressParam.NAME)
377      @DefaultValue(NamenodeAddressParam.DEFAULT)
378          final NamenodeAddressParam namenode,
379      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
380      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
381          final GetOpParam op,
382      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
383          final OffsetParam offset,
384      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
385          final LengthParam length,
386      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
387          final BufferSizeParam bufferSize
388      ) throws IOException, InterruptedException {
389
390    final String nnId = namenode.getValue();
391    init(ugi, delegation, nnId, path, op, offset, length, bufferSize);
392
393    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
394      @Override
395      public Response run() throws IOException {
396        return get(nnId, path.getAbsolutePath(), op, offset,
397                length, bufferSize);
398      }
399    });
400  }
401
402  private Response get(
403      final String nnId,
404      final String fullpath,
405      final GetOpParam op,
406      final OffsetParam offset,
407      final LengthParam length,
408      final BufferSizeParam bufferSize
409      ) throws IOException {
410    final DataNode datanode = (DataNode)context.getAttribute("datanode");
411    final Configuration conf = new Configuration(datanode.getConf());
412
413    switch(op.getValue()) {
414    case OPEN:
415    {
416      final int b = bufferSize.getValue(conf);
417      final DFSClient dfsclient = newDfsClient(nnId, conf);
418      HdfsDataInputStream in = null;
419      try {
420        in = new HdfsDataInputStream(dfsclient.open(fullpath, b, true));
421        in.seek(offset.getValue());
422      } catch(IOException ioe) {
423        IOUtils.cleanup(LOG, in);
424        IOUtils.cleanup(LOG, dfsclient);
425        throw ioe;
426      }
427      
428      final long n = length.getValue() != null ?
429        Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
430        in.getVisibleLength() - offset.getValue();
431
432      // jetty 6 reserves 12 bytes in the out buffer for chunked responses
433      // (file length > 2GB) which causes extremely poor performance when
434      // 12 bytes of the output spill into another buffer which results
435      // in a big and little write
436      int outBufferSize = response.getBufferSize();
437      if (n > Integer.MAX_VALUE) {
438        outBufferSize -= 12;
439      }
440      /**
441       * Allow the Web UI to perform an AJAX request to get the data.
442       */
443      return Response.ok(new OpenEntity(in, n, outBufferSize, dfsclient))
444          .type(MediaType.APPLICATION_OCTET_STREAM)
445          .header("Access-Control-Allow-Methods", "GET")
446          .header("Access-Control-Allow-Origin", "*")
447          .build();
448    }
449    case GETFILECHECKSUM:
450    {
451      MD5MD5CRC32FileChecksum checksum = null;
452      DFSClient dfsclient = newDfsClient(nnId, conf);
453      try {
454        checksum = dfsclient.getFileChecksum(fullpath, Long.MAX_VALUE);
455        dfsclient.close();
456        dfsclient = null;
457      } finally {
458        IOUtils.cleanup(LOG, dfsclient);
459      }
460      final String js = JsonUtil.toJsonString(checksum);
461      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
462    }
463    default:
464      throw new UnsupportedOperationException(op + " is not supported");
465    }
466  }
467
468  private static DFSClient newDfsClient(String nnId,
469                                        Configuration conf) throws IOException {
470    URI uri = URI.create(HdfsConstants.HDFS_URI_SCHEME + "://" + nnId);
471    return new DFSClient(uri, conf);
472  }
473}