@InterfaceAudience.LimitedPrivate(value="MapReduce") @InterfaceStability.Unstable public class DirectShuffleMergeManagerImpl<K,V> extends Object implements org.apache.hadoop.mapreduce.task.reduce.MergeManager<K,V>
Modifier and Type | Field and Description |
---|---|
protected org.apache.hadoop.mapred.MapOutputFile |
mapOutputFile |
Constructor and Description |
---|
DirectShuffleMergeManagerImpl(org.apache.hadoop.mapreduce.TaskAttemptID reduceId,
org.apache.hadoop.mapred.JobConf jobConf,
org.apache.hadoop.fs.FileSystem localFS,
org.apache.hadoop.mapred.Reporter reporter,
org.apache.hadoop.io.compress.CompressionCodec codec,
Class<? extends org.apache.hadoop.mapred.Reducer> combinerClass,
org.apache.hadoop.mapred.Task.CombineOutputCollector<K,V> combineCollector,
org.apache.hadoop.mapred.Counters.Counter spilledRecordsCounter,
org.apache.hadoop.mapred.Counters.Counter reduceCombineInputCounter,
org.apache.hadoop.mapred.Counters.Counter mergedMapOutputsCounter,
org.apache.hadoop.mapreduce.task.reduce.ExceptionReporter exceptionReporter,
org.apache.hadoop.util.Progress mergePhase,
org.apache.hadoop.mapred.MapOutputFile mapOutputFile) |
Modifier and Type | Method and Description |
---|---|
protected boolean |
canShuffleToMemory(long requestedSize) |
org.apache.hadoop.mapred.RawKeyValueIterator |
close() |
void |
closeInMemoryFile(DirectInMemoryOutput<K,V> mapOutput) |
void |
closeInMemoryMergedFile(DirectInMemoryOutput<K,V> mapOutput) |
void |
closeOnDiskFile(org.apache.hadoop.fs.FileStatus fileStatus) |
protected org.apache.hadoop.mapreduce.task.reduce.MergeThread<DirectInMemoryOutput<K,V>,K,V> |
createInMemoryMerger() |
org.apache.hadoop.mapreduce.task.reduce.MapOutput<K,V> |
reserve(org.apache.hadoop.mapreduce.TaskAttemptID mapId,
long requestedSize,
int fetcher) |
protected DirectInMemoryOutput<K,V> |
unconditionalReserve(org.apache.hadoop.mapreduce.TaskAttemptID mapId,
long requestedSize,
boolean primaryMapOutput)
Unconditional Reserve is used by the Memory-to-Memory thread
|
void |
waitForResource() |
public DirectShuffleMergeManagerImpl(org.apache.hadoop.mapreduce.TaskAttemptID reduceId, org.apache.hadoop.mapred.JobConf jobConf, org.apache.hadoop.fs.FileSystem localFS, org.apache.hadoop.mapred.Reporter reporter, org.apache.hadoop.io.compress.CompressionCodec codec, Class<? extends org.apache.hadoop.mapred.Reducer> combinerClass, org.apache.hadoop.mapred.Task.CombineOutputCollector<K,V> combineCollector, org.apache.hadoop.mapred.Counters.Counter spilledRecordsCounter, org.apache.hadoop.mapred.Counters.Counter reduceCombineInputCounter, org.apache.hadoop.mapred.Counters.Counter mergedMapOutputsCounter, org.apache.hadoop.mapreduce.task.reduce.ExceptionReporter exceptionReporter, org.apache.hadoop.util.Progress mergePhase, org.apache.hadoop.mapred.MapOutputFile mapOutputFile) throws IOException
IOException
protected org.apache.hadoop.mapreduce.task.reduce.MergeThread<DirectInMemoryOutput<K,V>,K,V> createInMemoryMerger()
public void waitForResource() throws InterruptedException
waitForResource
in interface org.apache.hadoop.mapreduce.task.reduce.MergeManager<K,V>
InterruptedException
protected boolean canShuffleToMemory(long requestedSize)
public org.apache.hadoop.mapreduce.task.reduce.MapOutput<K,V> reserve(org.apache.hadoop.mapreduce.TaskAttemptID mapId, long requestedSize, int fetcher) throws IOException
reserve
in interface org.apache.hadoop.mapreduce.task.reduce.MergeManager<K,V>
IOException
protected DirectInMemoryOutput<K,V> unconditionalReserve(org.apache.hadoop.mapreduce.TaskAttemptID mapId, long requestedSize, boolean primaryMapOutput)
public void closeInMemoryFile(DirectInMemoryOutput<K,V> mapOutput)
public void closeInMemoryMergedFile(DirectInMemoryOutput<K,V> mapOutput)
public void closeOnDiskFile(org.apache.hadoop.fs.FileStatus fileStatus)
Copyright © 2018 Apache Software Foundation. All Rights Reserved.