001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.namenode.snapshot;
019
020 import java.io.DataOutput;
021 import java.io.IOException;
022 import java.util.ArrayDeque;
023 import java.util.ArrayList;
024 import java.util.Collections;
025 import java.util.Deque;
026 import java.util.HashMap;
027 import java.util.Iterator;
028 import java.util.List;
029 import java.util.Map;
030
031 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
032 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
033 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
034 import org.apache.hadoop.hdfs.server.namenode.Content;
035 import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
036 import org.apache.hadoop.hdfs.server.namenode.INode;
037 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
038 import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
039 import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryWithQuota;
040 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
041 import org.apache.hadoop.hdfs.server.namenode.INodeMap;
042 import org.apache.hadoop.hdfs.server.namenode.INodeReference;
043 import org.apache.hadoop.hdfs.server.namenode.Quota;
044 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
045 import org.apache.hadoop.hdfs.util.Diff;
046 import org.apache.hadoop.hdfs.util.Diff.Container;
047 import org.apache.hadoop.hdfs.util.Diff.ListType;
048 import org.apache.hadoop.hdfs.util.Diff.UndoInfo;
049 import org.apache.hadoop.hdfs.util.ReadOnlyList;
050
051 import com.google.common.base.Preconditions;
052
053 /**
054 * The directory with snapshots. It maintains a list of snapshot diffs for
055 * storing snapshot data. When there are modifications to the directory, the old
056 * data is stored in the latest snapshot, if there is any.
057 */
058 public class INodeDirectoryWithSnapshot extends INodeDirectoryWithQuota {
059 /**
060 * The difference between the current state and a previous snapshot
061 * of the children list of an INodeDirectory.
062 */
063 static class ChildrenDiff extends Diff<byte[], INode> {
064 ChildrenDiff() {}
065
066 private ChildrenDiff(final List<INode> created, final List<INode> deleted) {
067 super(created, deleted);
068 }
069
070 /**
071 * Replace the given child from the created/deleted list.
072 * @return true if the child is replaced; false if the child is not found.
073 */
074 private final boolean replace(final ListType type,
075 final INode oldChild, final INode newChild) {
076 final List<INode> list = getList(type);
077 final int i = search(list, oldChild.getLocalNameBytes());
078 if (i < 0 || list.get(i).getId() != oldChild.getId()) {
079 return false;
080 }
081
082 final INode removed = list.set(i, newChild);
083 Preconditions.checkState(removed == oldChild);
084 return true;
085 }
086
087 private final boolean removeChild(ListType type, final INode child) {
088 final List<INode> list = getList(type);
089 final int i = searchIndex(type, child.getLocalNameBytes());
090 if (i >= 0 && list.get(i) == child) {
091 list.remove(i);
092 return true;
093 }
094 return false;
095 }
096
097 /** clear the created list */
098 private Quota.Counts destroyCreatedList(
099 final INodeDirectoryWithSnapshot currentINode,
100 final BlocksMapUpdateInfo collectedBlocks,
101 final List<INode> removedINodes) {
102 Quota.Counts counts = Quota.Counts.newInstance();
103 final List<INode> createdList = getList(ListType.CREATED);
104 for (INode c : createdList) {
105 c.computeQuotaUsage(counts, true);
106 c.destroyAndCollectBlocks(collectedBlocks, removedINodes);
107 // c should be contained in the children list, remove it
108 currentINode.removeChild(c);
109 }
110 createdList.clear();
111 return counts;
112 }
113
114 /** clear the deleted list */
115 private Quota.Counts destroyDeletedList(
116 final BlocksMapUpdateInfo collectedBlocks,
117 final List<INode> removedINodes) {
118 Quota.Counts counts = Quota.Counts.newInstance();
119 final List<INode> deletedList = getList(ListType.DELETED);
120 for (INode d : deletedList) {
121 d.computeQuotaUsage(counts, false);
122 d.destroyAndCollectBlocks(collectedBlocks, removedINodes);
123 }
124 deletedList.clear();
125 return counts;
126 }
127
128 /** Serialize {@link #created} */
129 private void writeCreated(DataOutput out) throws IOException {
130 final List<INode> created = getList(ListType.CREATED);
131 out.writeInt(created.size());
132 for (INode node : created) {
133 // For INode in created list, we only need to record its local name
134 byte[] name = node.getLocalNameBytes();
135 out.writeShort(name.length);
136 out.write(name);
137 }
138 }
139
140 /** Serialize {@link #deleted} */
141 private void writeDeleted(DataOutput out,
142 ReferenceMap referenceMap) throws IOException {
143 final List<INode> deleted = getList(ListType.DELETED);
144 out.writeInt(deleted.size());
145 for (INode node : deleted) {
146 FSImageSerialization.saveINode2Image(node, out, true, referenceMap);
147 }
148 }
149
150 /** Serialize to out */
151 private void write(DataOutput out, ReferenceMap referenceMap
152 ) throws IOException {
153 writeCreated(out);
154 writeDeleted(out, referenceMap);
155 }
156
157 /** Get the list of INodeDirectory contained in the deleted list */
158 private void getDirsInDeleted(List<INodeDirectory> dirList) {
159 for (INode node : getList(ListType.DELETED)) {
160 if (node.isDirectory()) {
161 dirList.add(node.asDirectory());
162 }
163 }
164 }
165
166 /**
167 * Interpret the diff and generate a list of {@link DiffReportEntry}.
168 * @param parentPath The relative path of the parent.
169 * @param parent The directory that the diff belongs to.
170 * @param fromEarlier True indicates {@code diff=later-earlier},
171 * False indicates {@code diff=earlier-later}
172 * @return A list of {@link DiffReportEntry} as the diff report.
173 */
174 public List<DiffReportEntry> generateReport(byte[][] parentPath,
175 INodeDirectoryWithSnapshot parent, boolean fromEarlier) {
176 List<DiffReportEntry> cList = new ArrayList<DiffReportEntry>();
177 List<DiffReportEntry> dList = new ArrayList<DiffReportEntry>();
178 int c = 0, d = 0;
179 List<INode> created = getList(ListType.CREATED);
180 List<INode> deleted = getList(ListType.DELETED);
181 byte[][] fullPath = new byte[parentPath.length + 1][];
182 System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
183 for (; c < created.size() && d < deleted.size(); ) {
184 INode cnode = created.get(c);
185 INode dnode = deleted.get(d);
186 if (cnode.compareTo(dnode.getLocalNameBytes()) == 0) {
187 fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
188 if (cnode.isSymlink() && dnode.isSymlink()) {
189 dList.add(new DiffReportEntry(DiffType.MODIFY, fullPath));
190 } else {
191 // must be the case: delete first and then create an inode with the
192 // same name
193 cList.add(new DiffReportEntry(DiffType.CREATE, fullPath));
194 dList.add(new DiffReportEntry(DiffType.DELETE, fullPath));
195 }
196 c++;
197 d++;
198 } else if (cnode.compareTo(dnode.getLocalNameBytes()) < 0) {
199 fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
200 cList.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
201 : DiffType.DELETE, fullPath));
202 c++;
203 } else {
204 fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
205 dList.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
206 : DiffType.CREATE, fullPath));
207 d++;
208 }
209 }
210 for (; d < deleted.size(); d++) {
211 fullPath[fullPath.length - 1] = deleted.get(d).getLocalNameBytes();
212 dList.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
213 : DiffType.CREATE, fullPath));
214 }
215 for (; c < created.size(); c++) {
216 fullPath[fullPath.length - 1] = created.get(c).getLocalNameBytes();
217 cList.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
218 : DiffType.DELETE, fullPath));
219 }
220 dList.addAll(cList);
221 return dList;
222 }
223 }
224
225 /**
226 * The difference of an {@link INodeDirectory} between two snapshots.
227 */
228 public static class DirectoryDiff extends
229 AbstractINodeDiff<INodeDirectory, INodeDirectoryAttributes, DirectoryDiff> {
230 /** The size of the children list at snapshot creation time. */
231 private final int childrenSize;
232 /** The children list diff. */
233 private final ChildrenDiff diff;
234
235 private DirectoryDiff(Snapshot snapshot, INodeDirectory dir) {
236 super(snapshot, null, null);
237
238 this.childrenSize = dir.getChildrenList(null).size();
239 this.diff = new ChildrenDiff();
240 }
241
242 /** Constructor used by FSImage loading */
243 DirectoryDiff(Snapshot snapshot, INodeDirectoryAttributes snapshotINode,
244 DirectoryDiff posteriorDiff, int childrenSize,
245 List<INode> createdList, List<INode> deletedList) {
246 super(snapshot, snapshotINode, posteriorDiff);
247 this.childrenSize = childrenSize;
248 this.diff = new ChildrenDiff(createdList, deletedList);
249 }
250
251 ChildrenDiff getChildrenDiff() {
252 return diff;
253 }
254
255 /** Is the inode the root of the snapshot? */
256 boolean isSnapshotRoot() {
257 return snapshotINode == snapshot.getRoot();
258 }
259
260 @Override
261 Quota.Counts combinePosteriorAndCollectBlocks(
262 final INodeDirectory currentDir, final DirectoryDiff posterior,
263 final BlocksMapUpdateInfo collectedBlocks,
264 final List<INode> removedINodes) {
265 final Quota.Counts counts = Quota.Counts.newInstance();
266 diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() {
267 /** Collect blocks for deleted files. */
268 @Override
269 public void process(INode inode) {
270 if (inode != null) {
271 inode.computeQuotaUsage(counts, false);
272 inode.destroyAndCollectBlocks(collectedBlocks, removedINodes);
273 }
274 }
275 });
276 return counts;
277 }
278
279 /**
280 * @return The children list of a directory in a snapshot.
281 * Since the snapshot is read-only, the logical view of the list is
282 * never changed although the internal data structure may mutate.
283 */
284 ReadOnlyList<INode> getChildrenList(final INodeDirectory currentDir) {
285 return new ReadOnlyList<INode>() {
286 private List<INode> children = null;
287
288 private List<INode> initChildren() {
289 if (children == null) {
290 final ChildrenDiff combined = new ChildrenDiff();
291 for(DirectoryDiff d = DirectoryDiff.this; d != null; d = d.getPosterior()) {
292 combined.combinePosterior(d.diff, null);
293 }
294 children = combined.apply2Current(ReadOnlyList.Util.asList(
295 currentDir.getChildrenList(null)));
296 }
297 return children;
298 }
299
300 @Override
301 public Iterator<INode> iterator() {
302 return initChildren().iterator();
303 }
304
305 @Override
306 public boolean isEmpty() {
307 return childrenSize == 0;
308 }
309
310 @Override
311 public int size() {
312 return childrenSize;
313 }
314
315 @Override
316 public INode get(int i) {
317 return initChildren().get(i);
318 }
319 };
320 }
321
322 /** @return the child with the given name. */
323 INode getChild(byte[] name, boolean checkPosterior,
324 INodeDirectory currentDir) {
325 for(DirectoryDiff d = this; ; d = d.getPosterior()) {
326 final Container<INode> returned = d.diff.accessPrevious(name);
327 if (returned != null) {
328 // the diff is able to determine the inode
329 return returned.getElement();
330 } else if (!checkPosterior) {
331 // Since checkPosterior is false, return null, i.e. not found.
332 return null;
333 } else if (d.getPosterior() == null) {
334 // no more posterior diff, get from current inode.
335 return currentDir.getChild(name, null);
336 }
337 }
338 }
339
340 @Override
341 public String toString() {
342 return super.toString() + " childrenSize=" + childrenSize + ", " + diff;
343 }
344
345 @Override
346 void write(DataOutput out, ReferenceMap referenceMap) throws IOException {
347 writeSnapshot(out);
348 out.writeInt(childrenSize);
349
350 // write snapshotINode
351 if (isSnapshotRoot()) {
352 out.writeBoolean(true);
353 } else {
354 out.writeBoolean(false);
355 if (snapshotINode != null) {
356 out.writeBoolean(true);
357 FSImageSerialization.writeINodeDirectoryAttributes(snapshotINode, out);
358 } else {
359 out.writeBoolean(false);
360 }
361 }
362 // Write diff. Node need to write poseriorDiff, since diffs is a list.
363 diff.write(out, referenceMap);
364 }
365
366 @Override
367 Quota.Counts destroyDiffAndCollectBlocks(INodeDirectory currentINode,
368 BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
369 // this diff has been deleted
370 Quota.Counts counts = Quota.Counts.newInstance();
371 counts.add(diff.destroyDeletedList(collectedBlocks, removedINodes));
372 return counts;
373 }
374 }
375
376 /** A list of directory diffs. */
377 public static class DirectoryDiffList
378 extends AbstractINodeDiffList<INodeDirectory, INodeDirectoryAttributes, DirectoryDiff> {
379
380 @Override
381 DirectoryDiff createDiff(Snapshot snapshot, INodeDirectory currentDir) {
382 return new DirectoryDiff(snapshot, currentDir);
383 }
384
385 @Override
386 INodeDirectoryAttributes createSnapshotCopy(INodeDirectory currentDir) {
387 return currentDir.isQuotaSet()?
388 new INodeDirectoryAttributes.CopyWithQuota(currentDir)
389 : new INodeDirectoryAttributes.SnapshotCopy(currentDir);
390 }
391
392 /** Replace the given child in the created/deleted list, if there is any. */
393 private boolean replaceChild(final ListType type, final INode oldChild,
394 final INode newChild) {
395 final List<DirectoryDiff> diffList = asList();
396 for(int i = diffList.size() - 1; i >= 0; i--) {
397 final ChildrenDiff diff = diffList.get(i).diff;
398 if (diff.replace(type, oldChild, newChild)) {
399 return true;
400 }
401 }
402 return false;
403 }
404
405 /** Remove the given child in the created/deleted list, if there is any. */
406 private boolean removeChild(final ListType type, final INode child) {
407 final List<DirectoryDiff> diffList = asList();
408 for(int i = diffList.size() - 1; i >= 0; i--) {
409 final ChildrenDiff diff = diffList.get(i).diff;
410 if (diff.removeChild(type, child)) {
411 return true;
412 }
413 }
414 return false;
415 }
416 }
417
418 /**
419 * Compute the difference between Snapshots.
420 *
421 * @param fromSnapshot Start point of the diff computation. Null indicates
422 * current tree.
423 * @param toSnapshot End point of the diff computation. Null indicates current
424 * tree.
425 * @param diff Used to capture the changes happening to the children. Note
426 * that the diff still represents (later_snapshot - earlier_snapshot)
427 * although toSnapshot can be before fromSnapshot.
428 * @return Whether changes happened between the startSnapshot and endSnaphsot.
429 */
430 boolean computeDiffBetweenSnapshots(Snapshot fromSnapshot,
431 Snapshot toSnapshot, ChildrenDiff diff) {
432 Snapshot earlier = fromSnapshot;
433 Snapshot later = toSnapshot;
434 if (Snapshot.ID_COMPARATOR.compare(fromSnapshot, toSnapshot) > 0) {
435 earlier = toSnapshot;
436 later = fromSnapshot;
437 }
438
439 boolean modified = diffs.changedBetweenSnapshots(earlier,
440 later);
441 if (!modified) {
442 return false;
443 }
444
445 final List<DirectoryDiff> difflist = diffs.asList();
446 final int size = difflist.size();
447 int earlierDiffIndex = Collections.binarySearch(difflist, earlier.getId());
448 int laterDiffIndex = later == null ? size : Collections
449 .binarySearch(difflist, later.getId());
450 earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1)
451 : earlierDiffIndex;
452 laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1)
453 : laterDiffIndex;
454
455 boolean dirMetadataChanged = false;
456 INodeDirectoryAttributes dirCopy = null;
457 for (int i = earlierDiffIndex; i < laterDiffIndex; i++) {
458 DirectoryDiff sdiff = difflist.get(i);
459 diff.combinePosterior(sdiff.diff, null);
460 if (dirMetadataChanged == false && sdiff.snapshotINode != null) {
461 if (dirCopy == null) {
462 dirCopy = sdiff.snapshotINode;
463 } else if (!dirCopy.metadataEquals(sdiff.snapshotINode)) {
464 dirMetadataChanged = true;
465 }
466 }
467 }
468
469 if (!diff.isEmpty() || dirMetadataChanged) {
470 return true;
471 } else if (dirCopy != null) {
472 for (int i = laterDiffIndex; i < size; i++) {
473 if (!dirCopy.metadataEquals(difflist.get(i).snapshotINode)) {
474 return true;
475 }
476 }
477 return !dirCopy.metadataEquals(this);
478 } else {
479 return false;
480 }
481 }
482
483 /** Diff list sorted by snapshot IDs, i.e. in chronological order. */
484 private final DirectoryDiffList diffs;
485
486 public INodeDirectoryWithSnapshot(INodeDirectory that) {
487 this(that, true, that instanceof INodeDirectoryWithSnapshot?
488 ((INodeDirectoryWithSnapshot)that).getDiffs(): null);
489 }
490
491 INodeDirectoryWithSnapshot(INodeDirectory that, boolean adopt,
492 DirectoryDiffList diffs) {
493 super(that, adopt, that.getNsQuota(), that.getDsQuota());
494 this.diffs = diffs != null? diffs: new DirectoryDiffList();
495 }
496
497 /** @return the last snapshot. */
498 public Snapshot getLastSnapshot() {
499 return diffs.getLastSnapshot();
500 }
501
502 /** @return the snapshot diff list. */
503 public DirectoryDiffList getDiffs() {
504 return diffs;
505 }
506
507 @Override
508 public INodeDirectoryAttributes getSnapshotINode(Snapshot snapshot) {
509 return diffs.getSnapshotINode(snapshot, this);
510 }
511
512 @Override
513 public INodeDirectoryWithSnapshot recordModification(final Snapshot latest,
514 final INodeMap inodeMap) throws QuotaExceededException {
515 if (isInLatestSnapshot(latest) && !shouldRecordInSrcSnapshot(latest)) {
516 return saveSelf2Snapshot(latest, null);
517 }
518 return this;
519 }
520
521 /** Save the snapshot copy to the latest snapshot. */
522 public INodeDirectoryWithSnapshot saveSelf2Snapshot(
523 final Snapshot latest, final INodeDirectory snapshotCopy)
524 throws QuotaExceededException {
525 diffs.saveSelf2Snapshot(latest, this, snapshotCopy);
526 return this;
527 }
528
529 @Override
530 public INode saveChild2Snapshot(final INode child, final Snapshot latest,
531 final INode snapshotCopy, final INodeMap inodeMap)
532 throws QuotaExceededException {
533 Preconditions.checkArgument(!child.isDirectory(),
534 "child is a directory, child=%s", child);
535 if (latest == null) {
536 return child;
537 }
538
539 final DirectoryDiff diff = diffs.checkAndAddLatestSnapshotDiff(latest, this);
540 if (diff.getChild(child.getLocalNameBytes(), false, this) != null) {
541 // it was already saved in the latest snapshot earlier.
542 return child;
543 }
544
545 diff.diff.modify(snapshotCopy, child);
546 return child;
547 }
548
549 @Override
550 public boolean addChild(INode inode, boolean setModTime, Snapshot latest,
551 final INodeMap inodeMap) throws QuotaExceededException {
552 ChildrenDiff diff = null;
553 Integer undoInfo = null;
554 if (isInLatestSnapshot(latest)) {
555 diff = diffs.checkAndAddLatestSnapshotDiff(latest, this).diff;
556 undoInfo = diff.create(inode);
557 }
558 final boolean added = super.addChild(inode, setModTime, null, inodeMap);
559 if (!added && undoInfo != null) {
560 diff.undoCreate(inode, undoInfo);
561 }
562 return added;
563 }
564
565 @Override
566 public boolean removeChild(INode child, Snapshot latest,
567 final INodeMap inodeMap) throws QuotaExceededException {
568 ChildrenDiff diff = null;
569 UndoInfo<INode> undoInfo = null;
570 // For a directory that is not a renamed node, if isInLatestSnapshot returns
571 // false, the directory is not in the latest snapshot, thus we do not need
572 // to record the removed child in any snapshot.
573 // For a directory that was moved/renamed, note that if the directory is in
574 // any of the previous snapshots, we will create a reference node for the
575 // directory while rename, and isInLatestSnapshot will return true in that
576 // scenario (if all previous snapshots have been deleted, isInLatestSnapshot
577 // still returns false). Thus if isInLatestSnapshot returns false, the
578 // directory node cannot be in any snapshot (not in current tree, nor in
579 // previous src tree). Thus we do not need to record the removed child in
580 // any snapshot.
581 if (isInLatestSnapshot(latest)) {
582 diff = diffs.checkAndAddLatestSnapshotDiff(latest, this).diff;
583 undoInfo = diff.delete(child);
584 }
585 final boolean removed = removeChild(child);
586 if (undoInfo != null) {
587 if (!removed) {
588 //remove failed, undo
589 diff.undoDelete(child, undoInfo);
590 }
591 }
592 return removed;
593 }
594
595 @Override
596 public void replaceChildFileInSnapshot(final INodeFile oldChild,
597 final INodeFile newChild) {
598 super.replaceChildFileInSnapshot(oldChild, newChild);
599 diffs.replaceChild(ListType.DELETED, oldChild, newChild);
600 diffs.replaceChild(ListType.CREATED, oldChild, newChild);
601 }
602
603 @Override
604 public void replaceChild(final INode oldChild, final INode newChild,
605 final INodeMap inodeMap) {
606 super.replaceChild(oldChild, newChild, inodeMap);
607 if (oldChild.getParentReference() != null && !newChild.isReference()) {
608 // oldChild is referred by a Reference node. Thus we are replacing the
609 // referred inode, e.g.,
610 // INodeFileWithSnapshot -> INodeFileUnderConstructionWithSnapshot
611 // in this case, we do not need to update the diff list
612 return;
613 } else {
614 diffs.replaceChild(ListType.CREATED, oldChild, newChild);
615 }
616 }
617
618 /**
619 * This method is usually called by the undo section of rename.
620 *
621 * Before calling this function, in the rename operation, we replace the
622 * original src node (of the rename operation) with a reference node (WithName
623 * instance) in both the children list and a created list, delete the
624 * reference node from the children list, and add it to the corresponding
625 * deleted list.
626 *
627 * To undo the above operations, we have the following steps in particular:
628 *
629 * <pre>
630 * 1) remove the WithName node from the deleted list (if it exists)
631 * 2) replace the WithName node in the created list with srcChild
632 * 3) add srcChild back as a child of srcParent. Note that we already add
633 * the node into the created list of a snapshot diff in step 2, we do not need
634 * to add srcChild to the created list of the latest snapshot.
635 * </pre>
636 *
637 * We do not need to update quota usage because the old child is in the
638 * deleted list before.
639 *
640 * @param oldChild
641 * The reference node to be removed/replaced
642 * @param newChild
643 * The node to be added back
644 * @param latestSnapshot
645 * The latest snapshot. Note this may not be the last snapshot in the
646 * {@link #diffs}, since the src tree of the current rename operation
647 * may be the dst tree of a previous rename.
648 * @throws QuotaExceededException should not throw this exception
649 */
650 public void undoRename4ScrParent(final INodeReference oldChild,
651 final INode newChild, Snapshot latestSnapshot)
652 throws QuotaExceededException {
653 diffs.removeChild(ListType.DELETED, oldChild);
654 diffs.replaceChild(ListType.CREATED, oldChild, newChild);
655 // pass null for inodeMap since the parent node will not get replaced when
656 // undoing rename
657 addChild(newChild, true, null, null);
658 }
659
660 /**
661 * Undo the rename operation for the dst tree, i.e., if the rename operation
662 * (with OVERWRITE option) removes a file/dir from the dst tree, add it back
663 * and delete possible record in the deleted list.
664 */
665 public void undoRename4DstParent(final INode deletedChild,
666 Snapshot latestSnapshot) throws QuotaExceededException {
667 boolean removeDeletedChild = diffs.removeChild(ListType.DELETED,
668 deletedChild);
669 // pass null for inodeMap since the parent node will not get replaced when
670 // undoing rename
671 final boolean added = addChild(deletedChild, true, removeDeletedChild ? null
672 : latestSnapshot, null);
673 // update quota usage if adding is successfully and the old child has not
674 // been stored in deleted list before
675 if (added && !removeDeletedChild) {
676 final Quota.Counts counts = deletedChild.computeQuotaUsage();
677 addSpaceConsumed(counts.get(Quota.NAMESPACE),
678 counts.get(Quota.DISKSPACE), false);
679 }
680 }
681
682 @Override
683 public ReadOnlyList<INode> getChildrenList(Snapshot snapshot) {
684 final DirectoryDiff diff = diffs.getDiff(snapshot);
685 return diff != null? diff.getChildrenList(this): super.getChildrenList(null);
686 }
687
688 @Override
689 public INode getChild(byte[] name, Snapshot snapshot) {
690 final DirectoryDiff diff = diffs.getDiff(snapshot);
691 return diff != null? diff.getChild(name, true, this): super.getChild(name, null);
692 }
693
694 @Override
695 public String toDetailString() {
696 return super.toDetailString() + ", " + diffs;
697 }
698
699 /**
700 * Get all the directories that are stored in some snapshot but not in the
701 * current children list. These directories are equivalent to the directories
702 * stored in the deletes lists.
703 */
704 public void getSnapshotDirectory(List<INodeDirectory> snapshotDir) {
705 for (DirectoryDiff sdiff : diffs) {
706 sdiff.getChildrenDiff().getDirsInDeleted(snapshotDir);
707 }
708 }
709
710 @Override
711 public Quota.Counts cleanSubtree(final Snapshot snapshot, Snapshot prior,
712 final BlocksMapUpdateInfo collectedBlocks,
713 final List<INode> removedINodes, final boolean countDiffChange)
714 throws QuotaExceededException {
715 Quota.Counts counts = Quota.Counts.newInstance();
716 Map<INode, INode> priorCreated = null;
717 Map<INode, INode> priorDeleted = null;
718 if (snapshot == null) { // delete the current directory
719 recordModification(prior, null);
720 // delete everything in created list
721 DirectoryDiff lastDiff = diffs.getLast();
722 if (lastDiff != null) {
723 counts.add(lastDiff.diff.destroyCreatedList(this, collectedBlocks,
724 removedINodes));
725 }
726 } else {
727 // update prior
728 prior = getDiffs().updatePrior(snapshot, prior);
729 // if there is a snapshot diff associated with prior, we need to record
730 // its original created and deleted list before deleting post
731 if (prior != null) {
732 DirectoryDiff priorDiff = this.getDiffs().getDiff(prior);
733 if (priorDiff != null && priorDiff.getSnapshot().equals(prior)) {
734 List<INode> cList = priorDiff.diff.getList(ListType.CREATED);
735 List<INode> dList = priorDiff.diff.getList(ListType.DELETED);
736 priorCreated = cloneDiffList(cList);
737 priorDeleted = cloneDiffList(dList);
738 }
739 }
740
741 counts.add(getDiffs().deleteSnapshotDiff(snapshot, prior, this,
742 collectedBlocks, removedINodes, countDiffChange));
743
744 // check priorDiff again since it may be created during the diff deletion
745 if (prior != null) {
746 DirectoryDiff priorDiff = this.getDiffs().getDiff(prior);
747 if (priorDiff != null && priorDiff.getSnapshot().equals(prior)) {
748 // For files/directories created between "prior" and "snapshot",
749 // we need to clear snapshot copies for "snapshot". Note that we must
750 // use null as prior in the cleanSubtree call. Files/directories that
751 // were created before "prior" will be covered by the later
752 // cleanSubtreeRecursively call.
753 if (priorCreated != null) {
754 // we only check the node originally in prior's created list
755 for (INode cNode : priorDiff.getChildrenDiff().getList(
756 ListType.CREATED)) {
757 if (priorCreated.containsKey(cNode)) {
758 counts.add(cNode.cleanSubtree(snapshot, null, collectedBlocks,
759 removedINodes, countDiffChange));
760 }
761 }
762 }
763
764 // When a directory is moved from the deleted list of the posterior
765 // diff to the deleted list of this diff, we need to destroy its
766 // descendants that were 1) created after taking this diff and 2)
767 // deleted after taking posterior diff.
768
769 // For files moved from posterior's deleted list, we also need to
770 // delete its snapshot copy associated with the posterior snapshot.
771
772 for (INode dNode : priorDiff.getChildrenDiff().getList(
773 ListType.DELETED)) {
774 if (priorDeleted == null || !priorDeleted.containsKey(dNode)) {
775 counts.add(cleanDeletedINode(dNode, snapshot, prior,
776 collectedBlocks, removedINodes, countDiffChange));
777 }
778 }
779 }
780 }
781 }
782 counts.add(cleanSubtreeRecursively(snapshot, prior, collectedBlocks,
783 removedINodes, priorDeleted, countDiffChange));
784
785 if (isQuotaSet()) {
786 this.addSpaceConsumed2Cache(-counts.get(Quota.NAMESPACE),
787 -counts.get(Quota.DISKSPACE));
788 }
789 return counts;
790 }
791
792 /**
793 * Clean an inode while we move it from the deleted list of post to the
794 * deleted list of prior.
795 * @param inode The inode to clean.
796 * @param post The post snapshot.
797 * @param prior The prior snapshot.
798 * @param collectedBlocks Used to collect blocks for later deletion.
799 * @return Quota usage update.
800 */
801 private static Quota.Counts cleanDeletedINode(INode inode,
802 final Snapshot post, final Snapshot prior,
803 final BlocksMapUpdateInfo collectedBlocks,
804 final List<INode> removedINodes, final boolean countDiffChange)
805 throws QuotaExceededException {
806 Quota.Counts counts = Quota.Counts.newInstance();
807 Deque<INode> queue = new ArrayDeque<INode>();
808 queue.addLast(inode);
809 while (!queue.isEmpty()) {
810 INode topNode = queue.pollFirst();
811 if (topNode instanceof INodeReference.WithName) {
812 INodeReference.WithName wn = (INodeReference.WithName) topNode;
813 if (wn.getLastSnapshotId() >= post.getId()) {
814 wn.cleanSubtree(post, prior, collectedBlocks, removedINodes,
815 countDiffChange);
816 }
817 // For DstReference node, since the node is not in the created list of
818 // prior, we should treat it as regular file/dir
819 } else if (topNode.isFile()
820 && topNode.asFile() instanceof FileWithSnapshot) {
821 FileWithSnapshot fs = (FileWithSnapshot) topNode.asFile();
822 counts.add(fs.getDiffs().deleteSnapshotDiff(post, prior,
823 topNode.asFile(), collectedBlocks, removedINodes, countDiffChange));
824 } else if (topNode.isDirectory()) {
825 INodeDirectory dir = topNode.asDirectory();
826 ChildrenDiff priorChildrenDiff = null;
827 if (dir instanceof INodeDirectoryWithSnapshot) {
828 // delete files/dirs created after prior. Note that these
829 // files/dirs, along with inode, were deleted right after post.
830 INodeDirectoryWithSnapshot sdir = (INodeDirectoryWithSnapshot) dir;
831 DirectoryDiff priorDiff = sdir.getDiffs().getDiff(prior);
832 if (priorDiff != null && priorDiff.getSnapshot().equals(prior)) {
833 priorChildrenDiff = priorDiff.getChildrenDiff();
834 counts.add(priorChildrenDiff.destroyCreatedList(sdir,
835 collectedBlocks, removedINodes));
836 }
837 }
838
839 for (INode child : dir.getChildrenList(prior)) {
840 if (priorChildrenDiff != null
841 && priorChildrenDiff.search(ListType.DELETED,
842 child.getLocalNameBytes()) != null) {
843 continue;
844 }
845 queue.addLast(child);
846 }
847 }
848 }
849 return counts;
850 }
851
852 @Override
853 public void destroyAndCollectBlocks(
854 final BlocksMapUpdateInfo collectedBlocks,
855 final List<INode> removedINodes) {
856 // destroy its diff list
857 for (DirectoryDiff diff : diffs) {
858 diff.destroyDiffAndCollectBlocks(this, collectedBlocks, removedINodes);
859 }
860 diffs.clear();
861 super.destroyAndCollectBlocks(collectedBlocks, removedINodes);
862 }
863
864 @Override
865 public final Quota.Counts computeQuotaUsage(Quota.Counts counts,
866 boolean useCache, int lastSnapshotId) {
867 if ((useCache && isQuotaSet()) || lastSnapshotId == Snapshot.INVALID_ID) {
868 return super.computeQuotaUsage(counts, useCache, lastSnapshotId);
869 }
870
871 Snapshot lastSnapshot = diffs.getSnapshotById(lastSnapshotId);
872
873 ReadOnlyList<INode> childrenList = getChildrenList(lastSnapshot);
874 for (INode child : childrenList) {
875 child.computeQuotaUsage(counts, useCache, lastSnapshotId);
876 }
877
878 counts.add(Quota.NAMESPACE, 1);
879 return counts;
880 }
881
882 @Override
883 public Quota.Counts computeQuotaUsage4CurrentDirectory(Quota.Counts counts) {
884 super.computeQuotaUsage4CurrentDirectory(counts);
885 for(DirectoryDiff d : diffs) {
886 for(INode deleted : d.getChildrenDiff().getList(ListType.DELETED)) {
887 deleted.computeQuotaUsage(counts, false, Snapshot.INVALID_ID);
888 }
889 }
890 counts.add(Quota.NAMESPACE, diffs.asList().size());
891 return counts;
892 }
893
894 @Override
895 public Content.Counts computeContentSummary(final Content.Counts counts) {
896 super.computeContentSummary(counts);
897 computeContentSummary4Snapshot(counts);
898 return counts;
899 }
900
901 private void computeContentSummary4Snapshot(final Content.Counts counts) {
902 for(DirectoryDiff d : diffs) {
903 for(INode deleted : d.getChildrenDiff().getList(ListType.DELETED)) {
904 deleted.computeContentSummary(counts);
905 }
906 }
907 counts.add(Content.DIRECTORY, diffs.asList().size());
908 }
909
910 private static Map<INode, INode> cloneDiffList(List<INode> diffList) {
911 if (diffList == null || diffList.size() == 0) {
912 return null;
913 }
914 Map<INode, INode> map = new HashMap<INode, INode>(diffList.size());
915 for (INode node : diffList) {
916 map.put(node, node);
917 }
918 return map;
919 }
920
921 /**
922 * Destroy a subtree under a DstReference node.
923 */
924 public static void destroyDstSubtree(INode inode, final Snapshot snapshot,
925 final Snapshot prior, final BlocksMapUpdateInfo collectedBlocks,
926 final List<INode> removedINodes) throws QuotaExceededException {
927 Preconditions.checkArgument(prior != null);
928 if (inode.isReference()) {
929 if (inode instanceof INodeReference.WithName && snapshot != null) {
930 // this inode has been renamed before the deletion of the DstReference
931 // subtree
932 inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes,
933 true);
934 } else {
935 // for DstReference node, continue this process to its subtree
936 destroyDstSubtree(inode.asReference().getReferredINode(), snapshot,
937 prior, collectedBlocks, removedINodes);
938 }
939 } else if (inode.isFile()) {
940 inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes, true);
941 } else if (inode.isDirectory()) {
942 Map<INode, INode> excludedNodes = null;
943 if (inode instanceof INodeDirectoryWithSnapshot) {
944 INodeDirectoryWithSnapshot sdir = (INodeDirectoryWithSnapshot) inode;
945
946 DirectoryDiffList diffList = sdir.getDiffs();
947 DirectoryDiff priorDiff = diffList.getDiff(prior);
948 if (priorDiff != null && priorDiff.getSnapshot().equals(prior)) {
949 List<INode> dList = priorDiff.diff.getList(ListType.DELETED);
950 excludedNodes = cloneDiffList(dList);
951 }
952
953 if (snapshot != null) {
954 diffList.deleteSnapshotDiff(snapshot, prior, sdir, collectedBlocks,
955 removedINodes, true);
956 }
957 priorDiff = diffList.getDiff(prior);
958 if (priorDiff != null && priorDiff.getSnapshot().equals(prior)) {
959 priorDiff.diff.destroyCreatedList(sdir, collectedBlocks,
960 removedINodes);
961 }
962 }
963 for (INode child : inode.asDirectory().getChildrenList(prior)) {
964 if (excludedNodes != null && excludedNodes.containsKey(child)) {
965 continue;
966 }
967 destroyDstSubtree(child, snapshot, prior, collectedBlocks,
968 removedINodes);
969 }
970 }
971 }
972 }