001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.namenode.snapshot;
019
020import java.io.DataInput;
021import java.io.DataOutput;
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.List;
026import java.util.Map;
027
028import org.apache.hadoop.hdfs.DFSUtil;
029import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
030import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
031import org.apache.hadoop.hdfs.server.namenode.INode;
032import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
033import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
034import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
035import org.apache.hadoop.hdfs.server.namenode.INodeReference;
036import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
037import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
038import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
039import org.apache.hadoop.hdfs.util.Diff.ListType;
040import org.apache.hadoop.hdfs.server.namenode.FSImageFormat.Loader;
041
042/**
043 * A helper class defining static methods for reading/writing snapshot related
044 * information from/to FSImage.
045 */
046public class SnapshotFSImageFormat {
047  public static FileDiffList loadFileDiffList(DataInput in,
048      FSImageFormat.Loader loader) throws IOException {
049    final int size = in.readInt();
050    if (size == -1) {
051      return null;
052    } else {
053      final FileDiffList diffs = new FileDiffList();
054      FileDiff posterior = null;
055      for(int i = 0; i < size; i++) {
056        final FileDiff d = loadFileDiff(posterior, in, loader);
057        diffs.addFirst(d);
058        posterior = d;
059      }
060      return diffs;
061    }
062  }
063
064  private static FileDiff loadFileDiff(FileDiff posterior, DataInput in,
065      FSImageFormat.Loader loader) throws IOException {
066    // 1. Read the id of the Snapshot root to identify the Snapshot
067    final Snapshot snapshot = loader.getSnapshot(in);
068
069    // 2. Load file size
070    final long fileSize = in.readLong();
071    
072    // 3. Load snapshotINode 
073    final INodeFileAttributes snapshotINode = in.readBoolean()?
074        loader.loadINodeFileAttributes(in): null;
075    
076    return new FileDiff(snapshot.getId(), snapshotINode, posterior, fileSize);
077  }
078
079  /**
080   * Load a node stored in the created list from fsimage.
081   * @param createdNodeName The name of the created node.
082   * @param parent The directory that the created list belongs to.
083   * @return The created node.
084   */
085  public static INode loadCreated(byte[] createdNodeName,
086      INodeDirectory parent) throws IOException {
087    // the INode in the created list should be a reference to another INode
088    // in posterior SnapshotDiffs or one of the current children
089    for (DirectoryDiff postDiff : parent.getDiffs()) {
090      final INode d = postDiff.getChildrenDiff().search(ListType.DELETED,
091          createdNodeName);
092      if (d != null) {
093        return d;
094      } // else go to the next SnapshotDiff
095    } 
096    // use the current child
097    INode currentChild = parent.getChild(createdNodeName,
098        Snapshot.CURRENT_STATE_ID);
099    if (currentChild == null) {
100      throw new IOException("Cannot find an INode associated with the INode "
101          + DFSUtil.bytes2String(createdNodeName)
102          + " in created list while loading FSImage.");
103    }
104    return currentChild;
105  }
106  
107  /**
108   * Load the created list from fsimage.
109   * @param parent The directory that the created list belongs to.
110   * @param in The {@link DataInput} to read.
111   * @return The created list.
112   */
113  private static List<INode> loadCreatedList(INodeDirectory parent,
114      DataInput in) throws IOException {
115    // read the size of the created list
116    int createdSize = in.readInt();
117    List<INode> createdList = new ArrayList<INode>(createdSize);
118    for (int i = 0; i < createdSize; i++) {
119      byte[] createdNodeName = FSImageSerialization.readLocalName(in);
120      INode created = loadCreated(createdNodeName, parent);
121      createdList.add(created);
122    }
123    return createdList;
124  }
125    
126  /**
127   * Load the deleted list from the fsimage.
128   * 
129   * @param parent The directory that the deleted list belongs to.
130   * @param createdList The created list associated with the deleted list in 
131   *                    the same Diff.
132   * @param in The {@link DataInput} to read.
133   * @param loader The {@link Loader} instance.
134   * @return The deleted list.
135   */
136  private static List<INode> loadDeletedList(INodeDirectory parent,
137      List<INode> createdList, DataInput in, FSImageFormat.Loader loader)
138      throws IOException {
139    int deletedSize = in.readInt();
140    List<INode> deletedList = new ArrayList<INode>(deletedSize);
141    for (int i = 0; i < deletedSize; i++) {
142      final INode deleted = loader.loadINodeWithLocalName(true, in, true);
143      deletedList.add(deleted);
144      // set parent: the parent field of an INode in the deleted list is not 
145      // useful, but set the parent here to be consistent with the original 
146      // fsdir tree.
147      deleted.setParent(parent);
148      if (deleted.isFile()) {
149        loader.updateBlocksMap(deleted.asFile());
150      }
151    }
152    return deletedList;
153  }
154  
155  /**
156   * Load snapshots and snapshotQuota for a Snapshottable directory.
157   *
158   * @param snapshottableParent
159   *          The snapshottable directory for loading.
160   * @param numSnapshots
161   *          The number of snapshots that the directory has.
162   * @param loader
163   *          The loader
164   */
165  public static void loadSnapshotList(
166      INodeDirectorySnapshottable snapshottableParent, int numSnapshots,
167      DataInput in, FSImageFormat.Loader loader) throws IOException {
168    for (int i = 0; i < numSnapshots; i++) {
169      // read snapshots
170      final Snapshot s = loader.getSnapshot(in);
171      s.getRoot().setParent(snapshottableParent);
172      snapshottableParent.addSnapshot(s);
173    }
174    int snapshotQuota = in.readInt();
175    snapshottableParent.setSnapshotQuota(snapshotQuota);
176  }
177  
178  /**
179   * Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot
180   * directory.
181   *
182   * @param dir
183   *          The snapshottable directory for loading.
184   * @param in
185   *          The {@link DataInput} instance to read.
186   * @param loader
187   *          The loader
188   */
189  public static void loadDirectoryDiffList(INodeDirectory dir,
190      DataInput in, FSImageFormat.Loader loader) throws IOException {
191    final int size = in.readInt();
192    if (dir.isWithSnapshot()) {
193      DirectoryDiffList diffs = dir.getDiffs();
194      for (int i = 0; i < size; i++) {
195        diffs.addFirst(loadDirectoryDiff(dir, in, loader));
196      }
197    }
198  }
199
200  /**
201   * Load the snapshotINode field of {@link AbstractINodeDiff}.
202   * @param snapshot The Snapshot associated with the {@link AbstractINodeDiff}.
203   * @param in The {@link DataInput} to read.
204   * @param loader The {@link Loader} instance that this loading procedure is
205   *               using.
206   * @return The snapshotINode.
207   */
208  private static INodeDirectoryAttributes loadSnapshotINodeInDirectoryDiff(
209      Snapshot snapshot, DataInput in, FSImageFormat.Loader loader)
210      throws IOException {
211    // read the boolean indicating whether snapshotINode == Snapshot.Root
212    boolean useRoot = in.readBoolean();      
213    if (useRoot) {
214      return snapshot.getRoot();
215    } else {
216      // another boolean is used to indicate whether snapshotINode is non-null
217      return in.readBoolean()? loader.loadINodeDirectoryAttributes(in): null;
218    }
219  }
220   
221  /**
222   * Load {@link DirectoryDiff} from fsimage.
223   * @param parent The directory that the SnapshotDiff belongs to.
224   * @param in The {@link DataInput} instance to read.
225   * @param loader The {@link Loader} instance that this loading procedure is 
226   *               using.
227   * @return A {@link DirectoryDiff}.
228   */
229  private static DirectoryDiff loadDirectoryDiff(INodeDirectory parent,
230      DataInput in, FSImageFormat.Loader loader) throws IOException {
231    // 1. Read the full path of the Snapshot root to identify the Snapshot
232    final Snapshot snapshot = loader.getSnapshot(in);
233
234    // 2. Load DirectoryDiff#childrenSize
235    int childrenSize = in.readInt();
236    
237    // 3. Load DirectoryDiff#snapshotINode 
238    INodeDirectoryAttributes snapshotINode = loadSnapshotINodeInDirectoryDiff(
239        snapshot, in, loader);
240    
241    // 4. Load the created list in SnapshotDiff#Diff
242    List<INode> createdList = loadCreatedList(parent, in);
243    
244    // 5. Load the deleted list in SnapshotDiff#Diff
245    List<INode> deletedList = loadDeletedList(parent, createdList, in, loader);
246    
247    // 6. Compose the SnapshotDiff
248    List<DirectoryDiff> diffs = parent.getDiffs().asList();
249    DirectoryDiff sdiff = new DirectoryDiff(snapshot.getId(), snapshotINode,
250        diffs.isEmpty() ? null : diffs.get(0), childrenSize, createdList,
251        deletedList, snapshotINode == snapshot.getRoot());
252    return sdiff;
253  }
254  
255
256  /** A reference map for fsimage serialization. */
257  public static class ReferenceMap {
258    /**
259     * Used to indicate whether the reference node itself has been saved
260     */
261    private final Map<Long, INodeReference.WithCount> referenceMap
262        = new HashMap<Long, INodeReference.WithCount>();
263    /**
264     * Used to record whether the subtree of the reference node has been saved 
265     */
266    private final Map<Long, Long> dirMap = new HashMap<Long, Long>();
267    
268    public boolean toProcessSubtree(long id) {
269      if (dirMap.containsKey(id)) {
270        return false;
271      } else {
272        dirMap.put(id, id);
273        return true;
274      }
275    }
276    
277    public INodeReference.WithCount loadINodeReferenceWithCount(
278        boolean isSnapshotINode, DataInput in, FSImageFormat.Loader loader
279        ) throws IOException {
280      final boolean firstReferred = in.readBoolean();
281
282      final INodeReference.WithCount withCount;
283      if (firstReferred) {
284        final INode referred = loader.loadINodeWithLocalName(isSnapshotINode,
285            in, true);
286        withCount = new INodeReference.WithCount(null, referred);
287        referenceMap.put(withCount.getId(), withCount);
288      } else {
289        final long id = in.readLong();
290        withCount = referenceMap.get(id);
291      }
292      return withCount;
293    }
294  }
295}