org.apache.crunch.impl.spark.collect

Class SparkCollectFactory

java.lang.Object

org.apache.crunch.impl.spark.collect.SparkCollectFactory

All Implemented Interfaces:

PCollectionFactory

public class SparkCollectFactory
extends Object
implements PCollectionFactory

Constructor Summary

Constructors

Constructor and Description
SparkCollectFactory()

Method Summary

Modifier and Type	Method and Description
<S,T> BaseDoCollection<T>	createDoCollection(String name, PCollectionImpl<S> parent, DoFn<S,T> fn, PType<T> type, ParallelDoOptions options)
<S,K,V> BaseDoTable<K,V>	createDoTable(String name, PCollectionImpl<S> parent, CombineFn<K,V> combineFn, DoFn<S,Pair<K,V>> fn, PTableType<K,V> type)
<S,K,V> BaseDoTable<K,V>	createDoTable(String name, PCollectionImpl<S> parent, DoFn<S,Pair<K,V>> fn, PTableType<K,V> type, ParallelDoOptions options)
<K,V> BaseGroupedTable<K,V>	createGroupedTable(PTableBase<K,V> parent, GroupingOptions groupingOptions)
<S> BaseInputCollection<S>	createInputCollection(Source<S> source, String named, DistributedPipeline pipeline, ParallelDoOptions doOpts)
<K,V> BaseInputTable<K,V>	createInputTable(TableSource<K,V> source, String named, DistributedPipeline pipeline, ParallelDoOptions doOpts)
<S> BaseUnionCollection<S>	createUnionCollection(List<? extends PCollectionImpl<S>> internal)
<K,V> PTable<K,V>	createUnionTable(List<PTableBase<K,V>> internal)

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

SparkCollectFactory

public SparkCollectFactory()

Method Detail

createInputCollection

public <S> BaseInputCollection<S> createInputCollection(Source<S> source,
                                                        String named,
                                                        DistributedPipeline pipeline,
                                                        ParallelDoOptions doOpts)

Specified by:

createInputCollection in interface PCollectionFactory

createInputTable

public <K,V> BaseInputTable<K,V> createInputTable(TableSource<K,V> source,
                                                  String named,
                                                  DistributedPipeline pipeline,
                                                  ParallelDoOptions doOpts)

Specified by:

createInputTable in interface PCollectionFactory

createUnionCollection

public <S> BaseUnionCollection<S> createUnionCollection(List<? extends PCollectionImpl<S>> internal)

Specified by:

createUnionCollection in interface PCollectionFactory

createDoCollection

public <S,T> BaseDoCollection<T> createDoCollection(String name,
                                                    PCollectionImpl<S> parent,
                                                    DoFn<S,T> fn,
                                                    PType<T> type,
                                                    ParallelDoOptions options)

Specified by:

createDoCollection in interface PCollectionFactory

createDoTable

public <S,K,V> BaseDoTable<K,V> createDoTable(String name,
                                              PCollectionImpl<S> parent,
                                              DoFn<S,Pair<K,V>> fn,
                                              PTableType<K,V> type,
                                              ParallelDoOptions options)

Specified by:

createDoTable in interface PCollectionFactory

createDoTable

public <S,K,V> BaseDoTable<K,V> createDoTable(String name,
                                              PCollectionImpl<S> parent,
                                              CombineFn<K,V> combineFn,
                                              DoFn<S,Pair<K,V>> fn,
                                              PTableType<K,V> type)

Specified by:

createDoTable in interface PCollectionFactory

createGroupedTable

public <K,V> BaseGroupedTable<K,V> createGroupedTable(PTableBase<K,V> parent,
                                                      GroupingOptions groupingOptions)

Specified by:

createGroupedTable in interface PCollectionFactory

createUnionTable

public <K,V> PTable<K,V> createUnionTable(List<PTableBase<K,V>> internal)

Specified by:

createUnionTable in interface PCollectionFactory