Java源码示例：org.apache.pig.PigWarning
示例1
@Override
public void visit(CastExpression cast) throws FrontendException{
byte inType = cast.getExpression().getType();
byte outType = cast.getType();
if(containsByteArrayOrEmtpyInSchema(cast.getExpression().getFieldSchema())){
long inUid = cast.getExpression().getFieldSchema().uid;
FuncSpec inLoadFunc = uid2LoadFuncMap.get(inUid);
if(inLoadFunc == null){
String msg = "Cannot resolve load function to use for casting from " +
DataType.findTypeName(inType) + " to " +
DataType.findTypeName(outType) + ". ";
msgCollector.collect(msg, MessageType.Warning,
PigWarning.NO_LOAD_FUNCTION_FOR_CASTING_BYTEARRAY);
}else {
cast.setFuncSpec(inLoadFunc);
}
}
}
示例2
/**
* Clear out the contents of the bag, both on disk and in memory.
* Any attempts to read after this is called will produce undefined
* results.
*/
@Override
public void clear() {
synchronized (mContents) {
mContents.clear();
if (mSpillFiles != null) {
for (int i = 0; i < mSpillFiles.size(); i++) {
boolean res = mSpillFiles.get(i).delete();
if (!res)
warn ("DefaultAbstractBag.clear: failed to delete " + mSpillFiles.get(i), PigWarning.DELETE_FAILED, null);
}
mSpillFiles.clear();
}
mSize = 0;
aggSampleTupleSize = 0;
sampled = 0;
// not changing spillableRegistered -- clear doesn't change that.
}
}
示例3
@Override
public Boolean exec(Tuple tuple) {
if (tuple == null || tuple.size() != 2) {
warn("invalid number of arguments to STARTSWITH", PigWarning.UDF_WARNING_1);
return null;
}
String argument = null;
String testAgainst = null;
try {
argument = (String) tuple.get(0);
testAgainst = (String) tuple.get(1);
return argument.startsWith(testAgainst);
} catch (NullPointerException npe) {
warn(npe.toString(), PigWarning.UDF_WARNING_2);
return null;
} catch (ClassCastException cce) {
warn(cce.toString(), PigWarning.UDF_WARNING_3);
return null;
} catch (ExecException ee) {
warn(ee.toString(), PigWarning.UDF_WARNING_4);
return null;
}
}
示例4
/**
* Method invoked on every tuple during foreach evaluation
* @param input tuple; first column is assumed to have the column to convert
* the second column is the string we search for
* the third is an optional column from where to start the search
* @exception java.io.IOException
* @return index of first occurrence, or null in case of processing error
*/
@Override
public Integer exec(Tuple input) throws IOException {
if (input == null || input.size() < 2) {
warn("invalid input tuple: "+input, PigWarning.UDF_WARNING_1);
return null;
}
try {
String str = (String)input.get(0);
String search = (String)input.get(1);
int fromIndex = 0;
if (input.size() >=3)
fromIndex = (Integer)input.get(2);
return str.indexOf(search, fromIndex);
} catch(Exception e){
warn("Failed to process input; error - " + e.getMessage(), PigWarning.UDF_WARNING_1);
return null;
}
}
示例5
@Override
public RequiredFieldResponse pushProjection(final RequiredFieldList rfl)
throws FrontendException {
requiredFieldList = rfl;
Schema newSchema = AvroStorageSchemaConversionUtilities
.newSchemaFromRequiredFieldList(schema, rfl);
if (newSchema != null) {
schema = newSchema;
setInputAvroSchema(schema);
return new RequiredFieldResponse(true);
} else {
log.warn("could not select fields subset " + rfl + "\n");
warn("could not select fields subset", PigWarning.UDF_WARNING_2);
return new RequiredFieldResponse(false);
}
}
示例6
@Override
public DataBag bytesToBag(byte[] b, ResourceFieldSchema schema) throws IOException {
if(b == null)
return null;
DataBag db;
try {
ByteArrayInputStream bis = new ByteArrayInputStream(b);
PushbackInputStream in = new PushbackInputStream(bis);
db = consumeBag(in, schema);
} catch (IOException e) {
LogUtils.warn(this, "Unable to interpret value " + Arrays.toString(b) + " in field being " +
"converted to type bag, caught ParseException <" +
e.getMessage() + "> field discarded",
PigWarning.FIELD_DISCARDED_TYPE_CONVERSION_FAILED, mLog);
return null;
}
return db;
}
示例7
@Override
public Double bytesToDouble(byte[] b) {
if(b == null || b.length == 0) {
return null;
}
try {
return Double.valueOf(new String(b));
} catch (NumberFormatException nfe) {
LogUtils.warn(this, "Unable to interpret value " + Arrays.toString(b) + " in field being " +
"converted to double, caught NumberFormatException <" +
nfe.getMessage() + "> field discarded",
PigWarning.FIELD_DISCARDED_TYPE_CONVERSION_FAILED, mLog);
return null;
}
}
示例8
@Override
public Float bytesToFloat(byte[] b) throws IOException {
if(b == null || b.length == 0) {
return null;
}
String s;
if (b.length > 0 && (b[b.length - 1] == 'F' || b[b.length - 1] == 'f')) {
s = new String(b, 0, b.length - 1);
} else {
s = new String(b);
}
try {
return Float.valueOf(s);
} catch (NumberFormatException nfe) {
LogUtils.warn(this, "Unable to interpret value " + Arrays.toString(b) + " in field being " +
"converted to float, caught NumberFormatException <" +
nfe.getMessage() + "> field discarded",
PigWarning.FIELD_DISCARDED_TYPE_CONVERSION_FAILED, mLog);
return null;
}
}
示例9
@Override
public DateTime bytesToDateTime(byte[] b) throws IOException {
if (b == null) {
return null;
}
try {
String dtStr = new String(b);
return ToDate.extractDateTime(dtStr);
} catch (IllegalArgumentException e) {
LogUtils.warn(this, "Unable to interpret value " + Arrays.toString(b) + " in field being " +
"converted to datetime, caught IllegalArgumentException <" +
e.getMessage() + "> field discarded",
PigWarning.FIELD_DISCARDED_TYPE_CONVERSION_FAILED, mLog);
return null;
}
}
示例10
@Override
public Map<String, Object> bytesToMap(byte[] b, ResourceFieldSchema fieldSchema) throws IOException {
if(b == null)
return null;
Map<String, Object> map;
try {
ByteArrayInputStream bis = new ByteArrayInputStream(b);
PushbackInputStream in = new PushbackInputStream(bis);
map = consumeMap(in, fieldSchema);
}
catch (IOException e) {
LogUtils.warn(this, "Unable to interpret value " + Arrays.toString(b) + " in field being " +
"converted to type map, caught ParseException <" +
e.getMessage() + "> field discarded",
PigWarning.FIELD_DISCARDED_TYPE_CONVERSION_FAILED, mLog);
return null;
}
return map;
}
示例11
@Override
public Tuple bytesToTuple(byte[] b, ResourceFieldSchema fieldSchema) throws IOException {
if(b == null)
return null;
Tuple t;
try {
ByteArrayInputStream bis = new ByteArrayInputStream(b);
PushbackInputStream in = new PushbackInputStream(bis);
t = consumeTuple(in, fieldSchema);
}
catch (IOException e) {
LogUtils.warn(this, "Unable to interpret value " + Arrays.toString(b) + " in field being " +
"converted to type tuple, caught ParseException <" +
e.getMessage() + "> field discarded",
PigWarning.FIELD_DISCARDED_TYPE_CONVERSION_FAILED, mLog);
return null;
}
return t;
}
示例12
@Override
public Boolean exec(Tuple tuple) {
if (tuple == null || tuple.size() != 2) {
warn("invalid number of arguments to ENDSWITH", PigWarning.UDF_WARNING_1);
return null;
}
String argument = null;
String testAgainst = null;
try {
argument = (String) tuple.get(0);
testAgainst = (String) tuple.get(1);
return argument.endsWith(testAgainst);
} catch (NullPointerException npe) {
warn(npe.toString(), PigWarning.UDF_WARNING_2);
return null;
} catch (ClassCastException cce) {
warn(cce.toString(), PigWarning.UDF_WARNING_3);
return null;
} catch (ExecException ee) {
warn(ee.toString(), PigWarning.UDF_WARNING_4);
return null;
}
}
示例13
/**
* Method invoked on every tuple during foreach evaluation
* @param input tuple; first column is assumed to have the column to convert
* @exception java.io.IOException
*/
@Override
public String exec(Tuple input) throws IOException {
if (input == null || input.size() < 3)
return null;
try{
String source = (String)input.get(0);
String target = (String)input.get(1);
String replacewith = (String)input.get(2);
return source.replaceAll(target, replacewith);
}catch(Exception e){
warn("Failed to process input; error - " + e.getMessage(), PigWarning.UDF_WARNING_1);
return null;
}
}
示例14
public static Double stringToDouble(String str) {
if (str == null) {
return null;
} else {
try {
return Double.parseDouble(str);
} catch (NumberFormatException e) {
LogUtils
.warn(
CastUtils.class,
"Unable to interpret value "
+ str
+ " in field being "
+ "converted to double, caught NumberFormatException <"
+ e.getMessage() + "> field discarded",
PigWarning.FIELD_DISCARDED_TYPE_CONVERSION_FAILED,
mLog);
return null;
}
}
}
示例15
public static Float stringToFloat(String str) {
if (str == null) {
return null;
} else {
try {
return Float.parseFloat(str);
} catch (NumberFormatException e) {
LogUtils
.warn(
CastUtils.class,
"Unable to interpret value "
+ str
+ " in field being "
+ "converted to float, caught NumberFormatException <"
+ e.getMessage() + "> field discarded",
PigWarning.FIELD_DISCARDED_TYPE_CONVERSION_FAILED,
mLog);
return null;
}
}
}
示例16
public static void logAggregate(Map<Enum, Long> aggMap, MessageType messageType, Log log) {
long nullCounterCount = aggMap.get(PigWarning.NULL_COUNTER_COUNT)==null?0 : aggMap.get(PigWarning.NULL_COUNTER_COUNT);
if (nullCounterCount!=0 && aggMap.size()>1) // PigWarning.NULL_COUNTER_COUNT is definitely in appMap
logMessage("Unable to retrieve hadoop counter for " + nullCounterCount +
" jobs, the number following warnings may not be correct", messageType, log);
for(Map.Entry<Enum, Long> e: aggMap.entrySet()) {
if (e.getKey() !=PigWarning.NULL_COUNTER_COUNT)
{
Long count = e.getValue();
if(count != null && count > 0) {
String message = "Encountered " + messageType + " " + e.getKey().toString() + " " + count + " time(s).";
logMessage(message, messageType, log);
}
}
}
}
示例17
/**
* Add i'th column from inpValue to objList
* @param objList
* @param inpValue
* @param i
* @throws ExecException
*/
private void addColumn(ArrayList<Object> objList, Tuple inpValue, int i)
throws ExecException {
try {
objList.add(inpValue.get(i));
} catch (IndexOutOfBoundsException ie) {
if(pigLogger != null) {
pigLogger.warn(this,"Attempt to access field " + i +
" which was not found in the input", PigWarning.ACCESSING_NON_EXISTENT_FIELD);
}
objList.add(null);
}
catch (NullPointerException npe) {
// the tuple is null, so a dereference should also produce a null
// there is a slight danger here that the Tuple implementation
// may have given the exception for a different reason but if we
// don't catch it, we will die and the most common case for the
// exception would be because the tuple is null
objList.add(null);
}
}
示例18
private void computeWarningAggregate(Map<String, Map<String, Long>> counterGroups, Map<Enum, Long> aggMap) {
for (Map<String, Long> counters : counterGroups.values()) {
for (Enum e : PigWarning.values()) {
if (counters.containsKey(e.toString())) {
if (aggMap.containsKey(e.toString())) {
Long currentCount = aggMap.get(e.toString());
currentCount = (currentCount == null ? 0 : currentCount);
if (counters != null) {
currentCount += counters.get(e.toString());
}
aggMap.put(e, currentCount);
} else {
aggMap.put(e, counters.get(e.toString()));
}
}
}
}
}
示例19
@Override
public String exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) {
return null;
}
try {
String str = (String) input.get(0);
if (str == null) return null;
if (str.length() == 0) return str;
char[] chars = str.toCharArray();
int lastIndex = chars.length-1;
for (int i=0; i<=lastIndex/2; i++) {
char c = chars[i];
chars[i] = chars[lastIndex-i];
chars[lastIndex-i] = c;
}
return new String(chars);
} catch (ExecException e) {
warn("Error reading input: " + e.getMessage(), PigWarning.UDF_WARNING_1);
return null;
}
}
示例20
@Override
public Boolean exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) return false;
try {
String str = (String)input.get(0);
if (str == null || str.length() == 0) return false;
Integer.parseInt(str);
} catch (NumberFormatException nfe) {
return false;
} catch (ClassCastException e) {
warn("Unable to cast input "+input.get(0)+" of class "+
input.get(0).getClass()+" to String", PigWarning.UDF_WARNING_1);
return false;
}
return true;
}
示例21
@Override
public Boolean exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) return false;
try {
String str = (String)input.get(0);
if (str == null || str.length() == 0) return false;
Float.parseFloat(str);
} catch (NumberFormatException nfe) {
return false;
} catch (ClassCastException e) {
warn("Unable to cast input "+input.get(0)+" of class "+
input.get(0).getClass()+" to String", PigWarning.UDF_WARNING_1);
return false;
}
return true;
}
示例22
@Override
public Boolean exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) return false;
try {
String str = (String)input.get(0);
if (str == null || str.length() == 0) return false;
Double.parseDouble(str);
} catch (NumberFormatException nfe) {
return false;
} catch (ClassCastException e) {
warn("Unable to cast input "+input.get(0)+" of class "+
input.get(0).getClass()+" to String", PigWarning.UDF_WARNING_1);
return false;
}
return true;
}
示例23
@Override
public Boolean exec(Tuple input) throws IOException {
if (input == null || input.size() == 0)
return false;
try {
String str = (String) input.get(0);
if (str == null || str.length() == 0)
return false;
if (str.startsWith("-"))
str = str.substring(1);
return str.matches("\\d+(\\.\\d+)?");
} catch (ClassCastException e) {
warn("Unable to cast input " + input.get(0) + " of class "
+ input.get(0).getClass() + " to String",
PigWarning.UDF_WARNING_1);
return false;
}
}
示例24
@Override
public Boolean exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) return false;
try {
String str = (String)input.get(0);
if (str == null || str.length() == 0) return false;
Long.parseLong(str);
} catch (NumberFormatException nfe) {
return false;
} catch (ClassCastException e) {
warn("Unable to cast input "+input.get(0)+" of class "+
input.get(0).getClass()+" to String", PigWarning.UDF_WARNING_1);
return false;
}
return true;
}
示例25
@Override
public String exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) {
return null;
}
try {
String str = (String) input.get(0);
if (str == null) return null;
if (str.length() == 0) return str;
return str.replaceFirst(" +$", "");
} catch (ExecException e) {
warn("Error reading input: " + e.getMessage(), PigWarning.UDF_WARNING_1);
return null;
}
}
示例26
@Override
public String exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) {
return null;
}
try {
String str = (String) input.get(0);
if (str == null) return null;
if (str.length() == 0) return str;
return Character.toUpperCase(str.charAt(0))+str.substring(1);
} catch (ExecException e) {
warn("Error reading input: " + e.getMessage(), PigWarning.UDF_WARNING_1);
return null;
}
}
示例27
@Override
public String exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) {
return null;
}
try {
String str = (String) input.get(0);
if (str == null) return null;
if (str.length() == 0) return str;
return str.replaceFirst("^ +", "");
} catch (ExecException e) {
warn("Error reading input: " + e.getMessage(), PigWarning.UDF_WARNING_1);
return null;
}
}
示例28
/**
* Method invoked on every tuple during foreach evaluation
* @param input tuple; first column is assumed to have the column to convert
* @exception java.io.IOException
*/
@Override
public String exec(Tuple input) throws IOException {
if (input == null || input.size() == 0)
return null;
try {
String str = (String)input.get(0);
return str.toLowerCase();
} catch(Exception e){
warn("Failed to process input; error - " + e.getMessage(), PigWarning.UDF_WARNING_1);
return null;
}
}
示例29
@Override
public String exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) {
return null;
}
try {
String str = (String) input.get(0);
if (str == null) return null;
if (str.length() == 0) return str;
return str.trim();
} catch (ExecException e) {
warn("Error reading input: " + e.getMessage(), PigWarning.UDF_WARNING_1);
return null;
}
}
示例30
/**
* Finds the last location of a substring in a given string.
* @param input tuple:<ol>
* <li>the string to process
* <li>the substring to find
* </ol>
* @exception java.io.IOException
* @return last location of substring, or null in case of processing errors.
*/
@Override
public Integer exec(Tuple input) throws IOException {
if (input == null || input.size() < 2)
return null;
try {
String str = (String)input.get(0);
String search = (String)input.get(1);
return str.lastIndexOf(search);
} catch(Exception e) {
warn("Failed to process input; error - " + e.getMessage(), PigWarning.UDF_WARNING_1);
return null;
}
}
