org.apache.pig.tools.pigstats
Class PigStatsUtil

java.lang.Object
  org.apache.pig.tools.pigstats.PigStatsUtil

public abstract class PigStatsUtil
extends Object

A utility class for Pig Statistics

Field Summary

static String	FS_COUNTER_GROUP
static String	HDFS_BYTES_READ
static String	HDFS_BYTES_WRITTEN
static String	MAP_INPUT_RECORDS
static String	MAP_OUTPUT_RECORDS
static String	MULTI_INPUTS_COUNTER_GROUP
static String	MULTI_INPUTS_RECORD_COUNTER
static String	MULTI_STORE_COUNTER_GROUP
static String	MULTI_STORE_RECORD_COUNTER
static String	REDUCE_INPUT_RECORDS
static String	REDUCE_OUTPUT_RECORDS
static String	TASK_COUNTER_GROUP

Constructor Summary

PigStatsUtil()

Method Summary

static void	accumulateStats(org.apache.hadoop.mapred.jobcontrol.JobControl jc) Updates the statistics after a patch of jobs is done
static JobStats	addNativeJobStats(PigStats ps, NativeMapReduceOper mr, boolean success)
static JobStats	addNativeJobStats(PigStats ps, NativeMapReduceOper mr, boolean success, Exception e)
static void	displayStatistics() Logs the statistics in the Pig log file at INFO level
static PigStats	getEmptyPigStats() Returns an empty PigStats object
static String	getMultiInputsCounterName(String fname, int index) Returns the counter name for the given input file name
static long	getMultiStoreCount(org.apache.hadoop.mapred.jobcontrol.Job job, org.apache.hadoop.mapred.JobClient jobClient, String counterName) Returns the count for the given counter name in the counter group 'MultiStoreCounters'
static String	getMultiStoreCounterName(POStore store) Returns the counter name for the given POStore
static PigStats	getPigStats(int code) Returns the PigStats with the given return code
static boolean	isTempFile(String fileName)
static void	setBackendException(org.apache.hadoop.mapred.jobcontrol.Job job, Exception e)
static void	setErrorCode(int code)
static void	setErrorMessage(String msg)
static void	setStatsMap(Map<String,List<PigStats>> statsMap)
static void	startCollection(PigContext pc, org.apache.hadoop.mapred.JobClient client, JobControlCompiler jcc, MROperPlan plan) Starts collecting statistics for the given MR plan
static void	stopCollection(boolean display) Stops collecting statistics for a MR plan
static void	updateJobMroMap(Map<org.apache.hadoop.mapred.jobcontrol.Job,MapReduceOper> jobMroMap) Updates the PigStats.JobGraph of the PigStats.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

MULTI_STORE_RECORD_COUNTER

public static final String MULTI_STORE_RECORD_COUNTER

See Also:

Constant Field Values

MULTI_STORE_COUNTER_GROUP

public static final String MULTI_STORE_COUNTER_GROUP

See Also:

Constant Field Values

TASK_COUNTER_GROUP

public static final String TASK_COUNTER_GROUP

See Also:

Constant Field Values

FS_COUNTER_GROUP

public static final String FS_COUNTER_GROUP

MAP_INPUT_RECORDS

public static final String MAP_INPUT_RECORDS

See Also:

Constant Field Values

MAP_OUTPUT_RECORDS

public static final String MAP_OUTPUT_RECORDS

See Also:

Constant Field Values

REDUCE_INPUT_RECORDS

public static final String REDUCE_INPUT_RECORDS

See Also:

Constant Field Values

REDUCE_OUTPUT_RECORDS

public static final String REDUCE_OUTPUT_RECORDS

See Also:

Constant Field Values

HDFS_BYTES_WRITTEN

public static final String HDFS_BYTES_WRITTEN

See Also:

Constant Field Values

HDFS_BYTES_READ

public static final String HDFS_BYTES_READ

See Also:

Constant Field Values

MULTI_INPUTS_RECORD_COUNTER

public static final String MULTI_INPUTS_RECORD_COUNTER

See Also:

Constant Field Values

MULTI_INPUTS_COUNTER_GROUP

public static final String MULTI_INPUTS_COUNTER_GROUP

See Also:

Constant Field Values

Constructor Detail

PigStatsUtil

public PigStatsUtil()

Method Detail

getMultiStoreCount

public static long getMultiStoreCount(org.apache.hadoop.mapred.jobcontrol.Job job,
                                      org.apache.hadoop.mapred.JobClient jobClient,
                                      String counterName)

Returns the count for the given counter name in the counter group 'MultiStoreCounters'

Parameters:

job - the MR job

jobClient - the Hadoop job client

counterName - the counter name

Returns:

the count of the given counter name

getMultiStoreCounterName

public static String getMultiStoreCounterName(POStore store)

Returns the counter name for the given POStore

Parameters:

store - the POStore

Returns:

the counter name

getMultiInputsCounterName

public static String getMultiInputsCounterName(String fname,
                                               int index)

Returns the counter name for the given input file name

Parameters:

fname - the input file name

Returns:

the counter name

startCollection

public static void startCollection(PigContext pc,
                                   org.apache.hadoop.mapred.JobClient client,
                                   JobControlCompiler jcc,
                                   MROperPlan plan)

Starts collecting statistics for the given MR plan

Parameters:

pc - the Pig context

client - the Hadoop job client

jcc - the job compiler

plan - the MR plan

stopCollection

public static void stopCollection(boolean display)

Stops collecting statistics for a MR plan

Parameters:

display - if true, log collected statistics in the Pig log file at INFO level

getEmptyPigStats

public static PigStats getEmptyPigStats()

Returns an empty PigStats object

Returns:

an empty PigStats object

getPigStats

public static PigStats getPigStats(int code)

Returns the PigStats with the given return code

Parameters:

code - the return code

Returns:

the PigStats with the given return code

displayStatistics

public static void displayStatistics()

Logs the statistics in the Pig log file at INFO level

updateJobMroMap

public static void updateJobMroMap(Map<org.apache.hadoop.mapred.jobcontrol.Job,MapReduceOper> jobMroMap)

Updates the PigStats.JobGraph of the PigStats. The initial PigStats.JobGraph is created without job ids using MROperPlan, before any job is submitted for execution. The PigStats.JobGraph then is updated with job ids after jobs are executed.

Parameters:

jobMroMap - the map that maps Jobs to MapReduceOpers

accumulateStats

public static void accumulateStats(org.apache.hadoop.mapred.jobcontrol.JobControl jc)

Updates the statistics after a patch of jobs is done

Parameters:

jc - the job control

setErrorMessage

public static void setErrorMessage(String msg)

setErrorCode

public static void setErrorCode(int code)

setBackendException

public static void setBackendException(org.apache.hadoop.mapred.jobcontrol.Job job,
                                       Exception e)

isTempFile

public static boolean isTempFile(String fileName)

addNativeJobStats

public static JobStats addNativeJobStats(PigStats ps,
                                         NativeMapReduceOper mr,
                                         boolean success)

setStatsMap

public static void setStatsMap(Map<String,List<PigStats>> statsMap)

addNativeJobStats

public static JobStats addNativeJobStats(PigStats ps,
                                         NativeMapReduceOper mr,
                                         boolean success,
                                         Exception e)