|
|||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | ||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |
java.lang.Object org.apache.crunch.io.impl.FileTargetImpl
public class FileTargetImpl
Nested Class Summary |
---|
Nested classes/interfaces inherited from interface org.apache.crunch.Target |
---|
Target.WriteMode |
Field Summary | |
---|---|
protected org.apache.hadoop.fs.Path |
path
|
Constructor Summary | |
---|---|
FileTargetImpl(org.apache.hadoop.fs.Path path,
Class<? extends org.apache.hadoop.mapreduce.lib.output.FileOutputFormat> outputFormatClass,
FileNamingScheme fileNamingScheme)
|
|
FileTargetImpl(org.apache.hadoop.fs.Path path,
Class<? extends org.apache.hadoop.mapreduce.lib.output.FileOutputFormat> outputFormatClass,
FileNamingScheme fileNamingScheme,
Map<String,String> extraConf)
|
Method Summary | ||
---|---|---|
boolean |
accept(OutputHandler handler,
PType<?> ptype)
Checks to see if this Target instance is compatible with the
given PType . |
|
|
asSourceTarget(PType<T> ptype)
Attempt to create the SourceTarget type that corresponds to this Target
for the given PType , if possible. |
|
protected void |
configureForMapReduce(org.apache.hadoop.mapreduce.Job job,
Class keyClass,
Class valueClass,
Class outputFormatClass,
org.apache.hadoop.fs.Path outputPath,
String name)
Deprecated. |
|
protected void |
configureForMapReduce(org.apache.hadoop.mapreduce.Job job,
Class keyClass,
Class valueClass,
FormatBundle formatBundle,
org.apache.hadoop.fs.Path outputPath,
String name)
|
|
void |
configureForMapReduce(org.apache.hadoop.mapreduce.Job job,
PType<?> ptype,
org.apache.hadoop.fs.Path outputPath,
String name)
|
|
boolean |
equals(Object other)
|
|
static int |
extractPartitionNumber(String reduceOutputFileName)
Extract the partition number from a raw reducer output filename. |
|
Converter<?,?,?,?> |
getConverter(PType<?> ptype)
Returns the Converter to use for mapping from the output PCollection
into the output values expected by this instance. |
|
protected org.apache.hadoop.fs.Path |
getDestFile(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.Path src,
org.apache.hadoop.fs.Path dir,
boolean mapOnlyJob)
|
|
FileNamingScheme |
getFileNamingScheme()
Get the naming scheme to be used for outputs being written to an output path. |
|
org.apache.hadoop.fs.Path |
getPath()
|
|
protected org.apache.hadoop.fs.Path |
getSourcePattern(org.apache.hadoop.fs.Path workingPath,
int index)
|
|
protected org.apache.hadoop.fs.Path |
getSuccessIndicator()
|
|
boolean |
handleExisting(Target.WriteMode strategy,
long lastModForSource,
org.apache.hadoop.conf.Configuration conf)
Apply the given WriteMode to this Target instance. |
|
void |
handleOutputs(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.Path workingPath,
int index)
Handles moving the output data for this target from a temporary location on the filesystem to its target path at the end of a MapReduce job. |
|
int |
hashCode()
|
|
protected static boolean |
isCompatible(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path)
|
|
Target |
outputConf(String key,
String value)
Adds the given key-value pair to the Configuration instance that is used to write
this Target . |
|
String |
toString()
|
Methods inherited from class java.lang.Object |
---|
clone, finalize, getClass, notify, notifyAll, wait, wait, wait |
Field Detail |
---|
protected final org.apache.hadoop.fs.Path path
Constructor Detail |
---|
public FileTargetImpl(org.apache.hadoop.fs.Path path, Class<? extends org.apache.hadoop.mapreduce.lib.output.FileOutputFormat> outputFormatClass, FileNamingScheme fileNamingScheme)
public FileTargetImpl(org.apache.hadoop.fs.Path path, Class<? extends org.apache.hadoop.mapreduce.lib.output.FileOutputFormat> outputFormatClass, FileNamingScheme fileNamingScheme, Map<String,String> extraConf)
Method Detail |
---|
public Target outputConf(String key, String value)
Target
Configuration
instance that is used to write
this Target
. Allows for multiple target outputs to re-use the same config keys with
different values when necessary.
outputConf
in interface Target
public void configureForMapReduce(org.apache.hadoop.mapreduce.Job job, PType<?> ptype, org.apache.hadoop.fs.Path outputPath, String name)
configureForMapReduce
in interface MapReduceTarget
@Deprecated protected void configureForMapReduce(org.apache.hadoop.mapreduce.Job job, Class keyClass, Class valueClass, Class outputFormatClass, org.apache.hadoop.fs.Path outputPath, String name)
protected void configureForMapReduce(org.apache.hadoop.mapreduce.Job job, Class keyClass, Class valueClass, FormatBundle formatBundle, org.apache.hadoop.fs.Path outputPath, String name)
public boolean accept(OutputHandler handler, PType<?> ptype)
Target
Target
instance is compatible with the
given PType
.
accept
in interface Target
handler
- The OutputHandler
that is managing the output for the jobptype
- The PType
to check
PType
,
false otherwisepublic Converter<?,?,?,?> getConverter(PType<?> ptype)
Target
Converter
to use for mapping from the output PCollection
into the output values expected by this instance.
getConverter
in interface Target
ptype
- The PType
of the data that is being written to this instance
Converter
for the output represented by this instancepublic void handleOutputs(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path workingPath, int index) throws IOException
PathTarget
handleOutputs
in interface PathTarget
conf
- The job Configuration
workingPath
- The temp directory that contains the output of the jobindex
- The index of this target for jobs that write multiple output files to a single directory
IOException
protected org.apache.hadoop.fs.Path getSuccessIndicator()
protected org.apache.hadoop.fs.Path getSourcePattern(org.apache.hadoop.fs.Path workingPath, int index)
public org.apache.hadoop.fs.Path getPath()
getPath
in interface PathTarget
protected static boolean isCompatible(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path)
protected org.apache.hadoop.fs.Path getDestFile(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dir, boolean mapOnlyJob) throws IOException
IOException
public static int extractPartitionNumber(String reduceOutputFileName)
reduceOutputFileName
- The raw reducer output file name
public FileNamingScheme getFileNamingScheme()
PathTarget
getFileNamingScheme
in interface PathTarget
public boolean equals(Object other)
equals
in class Object
public int hashCode()
hashCode
in class Object
public String toString()
toString
in class Object
public <T> SourceTarget<T> asSourceTarget(PType<T> ptype)
Target
SourceTarget
type that corresponds to this Target
for the given PType
, if possible. If it is not possible, return null
.
asSourceTarget
in interface Target
ptype
- The PType
to use in constructing the SourceTarget
SourceTarget
or null if such a SourceTarget
does not existpublic boolean handleExisting(Target.WriteMode strategy, long lastModForSource, org.apache.hadoop.conf.Configuration conf)
Target
WriteMode
to this Target
instance.
handleExisting
in interface Target
strategy
- The strategy for handling existing outputsconf
- The ever-useful Configuration
instance
|
|||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | ||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |