Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hdfs.server.namenode;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.OutputStream;
024import java.util.ArrayList;
025import java.util.Iterator;
026import java.util.List;
027import java.util.Map;
028
029import org.apache.commons.logging.Log;
030import org.apache.commons.logging.LogFactory;
031import org.apache.hadoop.HadoopIllegalArgumentException;
032import org.apache.hadoop.classification.InterfaceAudience;
033import org.apache.hadoop.fs.permission.AclEntry;
034import org.apache.hadoop.fs.permission.AclEntryScope;
035import org.apache.hadoop.fs.permission.AclEntryType;
036import org.apache.hadoop.fs.permission.FsAction;
037import org.apache.hadoop.fs.permission.FsPermission;
038import org.apache.hadoop.fs.permission.PermissionStatus;
039import org.apache.hadoop.hdfs.protocol.Block;
040import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
041import org.apache.hadoop.hdfs.protocolPB.PBHelper;
042import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
043import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
044import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
045import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext;
046import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SaverContext;
047import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
048import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry;
049import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection;
050import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
051import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.AclFeatureProto;
052import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
053import org.apache.hadoop.hdfs.util.ReadOnlyList;
054
055import com.google.common.base.Preconditions;
056import com.google.common.collect.ImmutableList;
057import com.google.protobuf.ByteString;
058
059@InterfaceAudience.Private
060public final class FSImageFormatPBINode {
061  private final static long USER_GROUP_STRID_MASK = (1 << 24) - 1;
062  private final static int USER_STRID_OFFSET = 40;
063  private final static int GROUP_STRID_OFFSET = 16;
064  private static final Log LOG = LogFactory.getLog(FSImageFormatPBINode.class);
065
066  private static final int ACL_ENTRY_NAME_MASK = (1 << 24) - 1;
067  private static final int ACL_ENTRY_NAME_OFFSET = 6;
068  private static final int ACL_ENTRY_TYPE_OFFSET = 3;
069  private static final int ACL_ENTRY_SCOPE_OFFSET = 5;
070  private static final int ACL_ENTRY_PERM_MASK = 7;
071  private static final int ACL_ENTRY_TYPE_MASK = 3;
072  private static final int ACL_ENTRY_SCOPE_MASK = 1;
073  private static final FsAction[] FSACTION_VALUES = FsAction.values();
074  private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES = AclEntryScope
075      .values();
076  private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES = AclEntryType
077      .values();
078
079  public final static class Loader {
080    public static PermissionStatus loadPermission(long id,
081        final String[] stringTable) {
082      short perm = (short) (id & ((1 << GROUP_STRID_OFFSET) - 1));
083      int gsid = (int) ((id >> GROUP_STRID_OFFSET) & USER_GROUP_STRID_MASK);
084      int usid = (int) ((id >> USER_STRID_OFFSET) & USER_GROUP_STRID_MASK);
085      return new PermissionStatus(stringTable[usid], stringTable[gsid],
086          new FsPermission(perm));
087    }
088
089    public static ImmutableList<AclEntry> loadAclEntries(
090        AclFeatureProto proto, final String[] stringTable) {
091      ImmutableList.Builder<AclEntry> b = ImmutableList.builder();
092      for (int v : proto.getEntriesList()) {
093        int p = v & ACL_ENTRY_PERM_MASK;
094        int t = (v >> ACL_ENTRY_TYPE_OFFSET) & ACL_ENTRY_TYPE_MASK;
095        int s = (v >> ACL_ENTRY_SCOPE_OFFSET) & ACL_ENTRY_SCOPE_MASK;
096        int nid = (v >> ACL_ENTRY_NAME_OFFSET) & ACL_ENTRY_NAME_MASK;
097        String name = stringTable[nid];
098        b.add(new AclEntry.Builder().setName(name)
099            .setPermission(FSACTION_VALUES[p])
100            .setScope(ACL_ENTRY_SCOPE_VALUES[s])
101            .setType(ACL_ENTRY_TYPE_VALUES[t]).build());
102      }
103      return b.build();
104    }
105
106    public static INodeDirectory loadINodeDirectory(INodeSection.INode n,
107        LoaderContext state) {
108      assert n.getType() == INodeSection.INode.Type.DIRECTORY;
109      INodeSection.INodeDirectory d = n.getDirectory();
110
111      final PermissionStatus permissions = loadPermission(d.getPermission(),
112          state.getStringTable());
113      final INodeDirectory dir = new INodeDirectory(n.getId(), n.getName()
114          .toByteArray(), permissions, d.getModificationTime());
115
116      final long nsQuota = d.getNsQuota(), dsQuota = d.getDsQuota();
117      if (nsQuota >= 0 || dsQuota >= 0) {
118        dir.addDirectoryWithQuotaFeature(nsQuota, dsQuota);
119      }
120
121      if (d.hasAcl()) {
122        dir.addAclFeature(new AclFeature(loadAclEntries(d.getAcl(),
123            state.getStringTable())));
124      }
125      return dir;
126    }
127
128    public static void updateBlocksMap(INodeFile file, BlockManager bm) {
129      // Add file->block mapping
130      final BlockInfo[] blocks = file.getBlocks();
131      if (blocks != null) {
132        for (int i = 0; i < blocks.length; i++) {
133          file.setBlock(i, bm.addBlockCollection(blocks[i], file));
134        }
135      }
136    }
137
138    private final FSDirectory dir;
139    private final FSNamesystem fsn;
140    private final FSImageFormatProtobuf.Loader parent;
141
142    Loader(FSNamesystem fsn, final FSImageFormatProtobuf.Loader parent) {
143      this.fsn = fsn;
144      this.dir = fsn.dir;
145      this.parent = parent;
146    }
147
148    void loadINodeDirectorySection(InputStream in) throws IOException {
149      final List<INodeReference> refList = parent.getLoaderContext()
150          .getRefList();
151      while (true) {
152        INodeDirectorySection.DirEntry e = INodeDirectorySection.DirEntry
153            .parseDelimitedFrom(in);
154        // note that in is a LimitedInputStream
155        if (e == null) {
156          break;
157        }
158        INodeDirectory p = dir.getInode(e.getParent()).asDirectory();
159        for (long id : e.getChildrenList()) {
160          INode child = dir.getInode(id);
161          addToParent(p, child);
162        }
163        for (int refId : e.getRefChildrenList()) {
164          INodeReference ref = refList.get(refId);
165          addToParent(p, ref);
166        }
167      }
168    }
169
170    void loadINodeSection(InputStream in) throws IOException {
171      INodeSection s = INodeSection.parseDelimitedFrom(in);
172      fsn.resetLastInodeId(s.getLastInodeId());
173      LOG.info("Loading " + s.getNumInodes() + " INodes.");
174      for (int i = 0; i < s.getNumInodes(); ++i) {
175        INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in);
176        if (p.getId() == INodeId.ROOT_INODE_ID) {
177          loadRootINode(p);
178        } else {
179          INode n = loadINode(p);
180          dir.addToInodeMap(n);
181        }
182      }
183    }
184
185    /**
186     * Load the under-construction files section, and update the lease map
187     */
188    void loadFilesUnderConstructionSection(InputStream in) throws IOException {
189      while (true) {
190        FileUnderConstructionEntry entry = FileUnderConstructionEntry
191            .parseDelimitedFrom(in);
192        if (entry == null) {
193          break;
194        }
195        // update the lease manager
196        INodeFile file = dir.getInode(entry.getInodeId()).asFile();
197        FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature();
198        Preconditions.checkState(uc != null); // file must be under-construction
199        fsn.leaseManager.addLease(uc.getClientName(), entry.getFullPath());
200      }
201    }
202
203    private void addToParent(INodeDirectory parent, INode child) {
204      if (parent == dir.rootDir && FSDirectory.isReservedName(child)) {
205        throw new HadoopIllegalArgumentException("File name \""
206            + child.getLocalName() + "\" is reserved. Please "
207            + " change the name of the existing file or directory to another "
208            + "name before upgrading to this release.");
209      }
210      // NOTE: This does not update space counts for parents
211      if (!parent.addChild(child)) {
212        return;
213      }
214      dir.cacheName(child);
215
216      if (child.isFile()) {
217        updateBlocksMap(child.asFile(), fsn.getBlockManager());
218      }
219    }
220
221    private INode loadINode(INodeSection.INode n) {
222      switch (n.getType()) {
223      case FILE:
224        return loadINodeFile(n);
225      case DIRECTORY:
226        return loadINodeDirectory(n, parent.getLoaderContext());
227      case SYMLINK:
228        return loadINodeSymlink(n);
229      default:
230        break;
231      }
232      return null;
233    }
234
235    private INodeFile loadINodeFile(INodeSection.INode n) {
236      assert n.getType() == INodeSection.INode.Type.FILE;
237      INodeSection.INodeFile f = n.getFile();
238      List<BlockProto> bp = f.getBlocksList();
239      short replication = (short) f.getReplication();
240      LoaderContext state = parent.getLoaderContext();
241
242      BlockInfo[] blocks = new BlockInfo[bp.size()];
243      for (int i = 0, e = bp.size(); i < e; ++i) {
244        blocks[i] = new BlockInfo(PBHelper.convert(bp.get(i)), replication);
245      }
246      final PermissionStatus permissions = loadPermission(f.getPermission(),
247          parent.getLoaderContext().getStringTable());
248
249      final INodeFile file = new INodeFile(n.getId(),
250          n.getName().toByteArray(), permissions, f.getModificationTime(),
251          f.getAccessTime(), blocks, replication, f.getPreferredBlockSize());
252
253      if (f.hasAcl()) {
254        file.addAclFeature(new AclFeature(loadAclEntries(f.getAcl(),
255            state.getStringTable())));
256      }
257
258      // under-construction information
259      if (f.hasFileUC()) {
260        INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
261        file.toUnderConstruction(uc.getClientName(), uc.getClientMachine(),
262            null);
263        if (blocks.length > 0) {
264          BlockInfo lastBlk = file.getLastBlock();
265          // replace the last block of file
266          file.setBlock(file.numBlocks() - 1, new BlockInfoUnderConstruction(
267              lastBlk, replication));
268        }
269      }
270      return file;
271    }
272
273
274    private INodeSymlink loadINodeSymlink(INodeSection.INode n) {
275      assert n.getType() == INodeSection.INode.Type.SYMLINK;
276      INodeSection.INodeSymlink s = n.getSymlink();
277      final PermissionStatus permissions = loadPermission(s.getPermission(),
278          parent.getLoaderContext().getStringTable());
279      INodeSymlink sym = new INodeSymlink(n.getId(), n.getName().toByteArray(),
280          permissions, s.getModificationTime(), s.getAccessTime(),
281          s.getTarget().toStringUtf8());
282      return sym;
283    }
284
285    private void loadRootINode(INodeSection.INode p) {
286      INodeDirectory root = loadINodeDirectory(p, parent.getLoaderContext());
287      final Quota.Counts q = root.getQuotaCounts();
288      final long nsQuota = q.get(Quota.NAMESPACE);
289      final long dsQuota = q.get(Quota.DISKSPACE);
290      if (nsQuota != -1 || dsQuota != -1) {
291        dir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
292      }
293      dir.rootDir.cloneModificationTime(root);
294      dir.rootDir.clonePermissionStatus(root);
295    }
296  }
297
298  public final static class Saver {
299    private static long buildPermissionStatus(INodeAttributes n,
300        final SaverContext.DeduplicationMap<String> stringMap) {
301      long userId = stringMap.getId(n.getUserName());
302      long groupId = stringMap.getId(n.getGroupName());
303      return ((userId & USER_GROUP_STRID_MASK) << USER_STRID_OFFSET)
304          | ((groupId & USER_GROUP_STRID_MASK) << GROUP_STRID_OFFSET)
305          | n.getFsPermissionShort();
306    }
307
308    private static AclFeatureProto.Builder buildAclEntries(AclFeature f,
309        final SaverContext.DeduplicationMap<String> map) {
310      AclFeatureProto.Builder b = AclFeatureProto.newBuilder();
311      for (AclEntry e : f.getEntries()) {
312        int v = ((map.getId(e.getName()) & ACL_ENTRY_NAME_MASK) << ACL_ENTRY_NAME_OFFSET)
313            | (e.getType().ordinal() << ACL_ENTRY_TYPE_OFFSET)
314            | (e.getScope().ordinal() << ACL_ENTRY_SCOPE_OFFSET)
315            | (e.getPermission().ordinal());
316        b.addEntries(v);
317      }
318      return b;
319    }
320
321    public static INodeSection.INodeFile.Builder buildINodeFile(
322        INodeFileAttributes file, final SaverContext state) {
323      INodeSection.INodeFile.Builder b = INodeSection.INodeFile.newBuilder()
324          .setAccessTime(file.getAccessTime())
325          .setModificationTime(file.getModificationTime())
326          .setPermission(buildPermissionStatus(file, state.getStringMap()))
327          .setPreferredBlockSize(file.getPreferredBlockSize())
328          .setReplication(file.getFileReplication());
329
330      AclFeature f = file.getAclFeature();
331      if (f != null) {
332        b.setAcl(buildAclEntries(f, state.getStringMap()));
333      }
334      return b;
335    }
336
337    public static INodeSection.INodeDirectory.Builder buildINodeDirectory(
338        INodeDirectoryAttributes dir, final SaverContext state) {
339      Quota.Counts quota = dir.getQuotaCounts();
340      INodeSection.INodeDirectory.Builder b = INodeSection.INodeDirectory
341          .newBuilder().setModificationTime(dir.getModificationTime())
342          .setNsQuota(quota.get(Quota.NAMESPACE))
343          .setDsQuota(quota.get(Quota.DISKSPACE))
344          .setPermission(buildPermissionStatus(dir, state.getStringMap()));
345
346      AclFeature f = dir.getAclFeature();
347      if (f != null) {
348        b.setAcl(buildAclEntries(f, state.getStringMap()));
349      }
350      return b;
351    }
352
353    private final FSNamesystem fsn;
354    private final FileSummary.Builder summary;
355    private final SaveNamespaceContext context;
356    private final FSImageFormatProtobuf.Saver parent;
357
358    Saver(FSImageFormatProtobuf.Saver parent, FileSummary.Builder summary) {
359      this.parent = parent;
360      this.summary = summary;
361      this.context = parent.getContext();
362      this.fsn = context.getSourceNamesystem();
363    }
364
365    void serializeINodeDirectorySection(OutputStream out) throws IOException {
366      Iterator<INodeWithAdditionalFields> iter = fsn.getFSDirectory()
367          .getINodeMap().getMapIterator();
368      final ArrayList<INodeReference> refList = parent.getSaverContext()
369          .getRefList();
370      int i = 0;
371      while (iter.hasNext()) {
372        INodeWithAdditionalFields n = iter.next();
373        if (!n.isDirectory()) {
374          continue;
375        }
376
377        ReadOnlyList<INode> children = n.asDirectory().getChildrenList(
378            Snapshot.CURRENT_STATE_ID);
379        if (children.size() > 0) {
380          INodeDirectorySection.DirEntry.Builder b = INodeDirectorySection.
381              DirEntry.newBuilder().setParent(n.getId());
382          for (INode inode : children) {
383            if (!inode.isReference()) {
384              b.addChildren(inode.getId());
385            } else {
386              refList.add(inode.asReference());
387              b.addRefChildren(refList.size() - 1);
388            }
389          }
390          INodeDirectorySection.DirEntry e = b.build();
391          e.writeDelimitedTo(out);
392        }
393
394        ++i;
395        if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) {
396          context.checkCancelled();
397        }
398      }
399      parent.commitSection(summary,
400          FSImageFormatProtobuf.SectionName.INODE_DIR);
401    }
402
403    void serializeINodeSection(OutputStream out) throws IOException {
404      INodeMap inodesMap = fsn.dir.getINodeMap();
405
406      INodeSection.Builder b = INodeSection.newBuilder()
407          .setLastInodeId(fsn.getLastInodeId()).setNumInodes(inodesMap.size());
408      INodeSection s = b.build();
409      s.writeDelimitedTo(out);
410
411      int i = 0;
412      Iterator<INodeWithAdditionalFields> iter = inodesMap.getMapIterator();
413      while (iter.hasNext()) {
414        INodeWithAdditionalFields n = iter.next();
415        save(out, n);
416        ++i;
417        if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) {
418          context.checkCancelled();
419        }
420      }
421      parent.commitSection(summary, FSImageFormatProtobuf.SectionName.INODE);
422    }
423
424    void serializeFilesUCSection(OutputStream out) throws IOException {
425      Map<String, INodeFile> ucMap = fsn.getFilesUnderConstruction();
426      for (Map.Entry<String, INodeFile> entry : ucMap.entrySet()) {
427        String path = entry.getKey();
428        INodeFile file = entry.getValue();
429        FileUnderConstructionEntry.Builder b = FileUnderConstructionEntry
430            .newBuilder().setInodeId(file.getId()).setFullPath(path);
431        FileUnderConstructionEntry e = b.build();
432        e.writeDelimitedTo(out);
433      }
434      parent.commitSection(summary,
435          FSImageFormatProtobuf.SectionName.FILES_UNDERCONSTRUCTION);
436    }
437
438    private void save(OutputStream out, INode n) throws IOException {
439      if (n.isDirectory()) {
440        save(out, n.asDirectory());
441      } else if (n.isFile()) {
442        save(out, n.asFile());
443      } else if (n.isSymlink()) {
444        save(out, n.asSymlink());
445      }
446    }
447
448    private void save(OutputStream out, INodeDirectory n) throws IOException {
449      INodeSection.INodeDirectory.Builder b = buildINodeDirectory(n,
450          parent.getSaverContext());
451      INodeSection.INode r = buildINodeCommon(n)
452          .setType(INodeSection.INode.Type.DIRECTORY).setDirectory(b).build();
453      r.writeDelimitedTo(out);
454    }
455
456    private void save(OutputStream out, INodeFile n) throws IOException {
457      INodeSection.INodeFile.Builder b = buildINodeFile(n,
458          parent.getSaverContext());
459
460      for (Block block : n.getBlocks()) {
461        b.addBlocks(PBHelper.convert(block));
462      }
463
464      FileUnderConstructionFeature uc = n.getFileUnderConstructionFeature();
465      if (uc != null) {
466        INodeSection.FileUnderConstructionFeature f =
467            INodeSection.FileUnderConstructionFeature
468            .newBuilder().setClientName(uc.getClientName())
469            .setClientMachine(uc.getClientMachine()).build();
470        b.setFileUC(f);
471      }
472
473      INodeSection.INode r = buildINodeCommon(n)
474          .setType(INodeSection.INode.Type.FILE).setFile(b).build();
475      r.writeDelimitedTo(out);
476    }
477
478    private void save(OutputStream out, INodeSymlink n) throws IOException {
479      SaverContext state = parent.getSaverContext();
480      INodeSection.INodeSymlink.Builder b = INodeSection.INodeSymlink
481          .newBuilder()
482          .setPermission(buildPermissionStatus(n, state.getStringMap()))
483          .setTarget(ByteString.copyFrom(n.getSymlink()))
484          .setModificationTime(n.getModificationTime())
485          .setAccessTime(n.getAccessTime());
486
487      INodeSection.INode r = buildINodeCommon(n)
488          .setType(INodeSection.INode.Type.SYMLINK).setSymlink(b).build();
489      r.writeDelimitedTo(out);
490    }
491
492    private final INodeSection.INode.Builder buildINodeCommon(INode n) {
493      return INodeSection.INode.newBuilder()
494          .setId(n.getId())
495          .setName(ByteString.copyFrom(n.getLocalNameBytes()));
496    }
497  }
498
499  private FSImageFormatPBINode() {
500  }
501}