hive 函数
year()
用法
select year('2022-02-01')
返回值
2022
源码地址
org.apache.hadoop.hive.ql.udf.UDFYear
源代码
/**
* UDFYear.
*
*/
// @Description 注解为函数的描述与用法
@Description(name = "year",
value = "_FUNC_(param) - Returns the year component of the date/timestamp/interval",
extended = "param can be one of:\n"
+ "1. A string in the format of 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'.\n"
+ "2. A date value\n"
+ "3. A timestamp value\n"
+ "4. A year-month interval value"
+ "Example:\n "
+ " > SELECT _FUNC_('2009-07-30') FROM src LIMIT 1;\n" + " 2009")
@VectorizedExpressions({VectorUDFYearDate.class, VectorUDFYearString.class, VectorUDFYearTimestamp.class})
// TODO NDV (number of distinct values) 注解是预设的一个估计值，帮助查询优化器更好的评估该列的数据分布
@NDV(maxNdv = 20) // although technically its unbounded, its unlikely we will ever see ndv > 20
public class UDFYear extends GenericUDF {
private transient ObjectInspectorConverters.Converter[] converters =
new ObjectInspectorConverters.Converter[1];
private transient PrimitiveObjectInspector.PrimitiveCategory[] inputTypes =
new PrimitiveObjectInspector.PrimitiveCategory[1];
private final IntWritable output = new IntWritable();
private final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
// 函数输入参数个数检验
checkArgsSize(arguments, 1, 1);
// 函数输入参数数据类型检验 Primitive 是否是原始数据类型
checkArgPrimitive(arguments, 0);
// 对传入的参数 进行类型转换
switch (((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()) {
case INTERVAL_YEAR_MONTH:
inputTypes[0] = PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_YEAR_MONTH;
converters[0] = ObjectInspectorConverters.getConverter(
arguments[0], PrimitiveObjectInspectorFactory.writableHiveIntervalYearMonthObjectInspector);
break;
case STRING:
case CHAR:
case VARCHAR:
case DATE:
case TIMESTAMP:
case TIMESTAMPLOCALTZ:
case VOID:
obtainDateConverter(arguments, 0, inputTypes, converters);
break;
default:
// build error message
StringBuilder sb = new StringBuilder();
sb.append(getFuncName());
sb.append(" does not take ");
sb.append(((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory());
sb.append(" type");
throw new UDFArgumentTypeException(0, sb.toString());
}
// 返回一个输出类型对象检查器
ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector;
return outputOI;
}
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
switch (inputTypes[0]) {
case INTERVAL_YEAR_MONTH:
HiveIntervalYearMonth intervalYearMonth = getIntervalYearMonthValue(arguments, 0, inputTypes, converters);
if (intervalYearMonth == null) {
return null;
}
output.set(intervalYearMonth.getYears());
break;
case STRING:
case CHAR:
case VARCHAR:
case DATE:
case TIMESTAMP:
case TIMESTAMPLOCALTZ:
case VOID:
// 转换成date 类型然后再使用calendar 取年
Date date = getDateValue(arguments, 0, inputTypes, converters);
if (date == null) {
return null;
}
calendar.setTimeInMillis(date.toEpochMilli());
output.set(calendar.get(Calendar.YEAR));
}
return output;
}
// 返回函数的名称 用于
@Override
protected String getFuncName() {
return "year";
}
@Override
public String getDisplayString(String[] children) {
return getStandardDisplayString(getFuncName(), children);
}
}
源码分析
- 整体上year() 函数比较简单，主要的方法有两个 initialize(ObjectInspector[] arguments) 与 evaluate(DeferredObject[] arguments)
- initialize(ObjectInspector[] arguments) 进行初始化的一些操作
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
// 函数输入参数个数检验
checkArgsSize(arguments, 1, 1);
// 函数输入参数数据类型检验 Primitive 是否是原始数据类型
checkArgPrimitive(arguments, 0);
// 对传入的参数 进行类型转换
switch (((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()) {
case INTERVAL_YEAR_MONTH:
inputTypes[0] = PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_YEAR_MONTH;
converters[0] = ObjectInspectorConverters.getConverter(
arguments[0], PrimitiveObjectInspectorFactory.writableHiveIntervalYearMonthObjectInspector);
break;
case STRING:
case CHAR:
case VARCHAR:
case DATE:
case TIMESTAMP:
case TIMESTAMPLOCALTZ:
case VOID:
obtainDateConverter(arguments, 0, inputTypes, converters);
break;
default:
// build error message
StringBuilder sb = new StringBuilder();
sb.append(getFuncName());
sb.append(" does not take ");
sb.append(((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory());
sb.append(" type");
throw new UDFArgumentTypeException(0, sb.toString());
}
// 返回一个输出类型对象检查器
ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector;
return outputOI;
}
整体上就是对输入的参数进行校验与转换并返回一个输出类型的对象检查器
3. evaluate(DeferredObject[] arguments) 是 year() udf函数的核心操作‘
public Object evaluate(DeferredObject[] arguments) throws HiveException {
switch (inputTypes[0]) {
case INTERVAL_YEAR_MONTH:
HiveIntervalYearMonth intervalYearMonth = getIntervalYearMonthValue(arguments, 0, inputTypes, converters);
if (intervalYearMonth == null) {
return null;
}
output.set(intervalYearMonth.getYears());
break;
case STRING:
case CHAR:
case VARCHAR:
case DATE:
case TIMESTAMP:
case TIMESTAMPLOCALTZ:
case VOID:
Date date = getDateValue(arguments, 0, inputTypes, converters);
if (date == null) {
return null;
}
calendar.setTimeInMillis(date.toEpochMilli());
output.set(calendar.get(Calendar.YEAR));
}
return output;
}
对输入的日期获取年
当输入的类型是 INTERVAL_YEAR_MONTH 的时候 直接进 getIntervalYearMonthValue(arguments, 0, inputTypes, converters)
// org.apache.hadoop.hive.ql.udf.generic.GenericUDF
protected HiveIntervalYearMonth getIntervalYearMonthValue(DeferredObject[] arguments, int i,
PrimitiveCategory[] inputTypes,
Converter[] converters) throws HiveException {
Object obj;
// 对输入的参数进行校验，当是null的时候直接返回
if ((obj = arguments[i].get()) == null) {
return null;
}
HiveIntervalYearMonth intervalYearMonth;
switch (inputTypes[i]) {
case STRING:
case VARCHAR:
case CHAR:
String intervalYearMonthStr = converters[i].convert(obj).toString();
intervalYearMonth = HiveIntervalYearMonth.valueOf(intervalYearMonthStr);
break;
case INTERVAL_YEAR_MONTH:
Object writableValue = converters[i].convert(obj);
intervalYearMonth = ((HiveIntervalYearMonthWritable) writableValue).getHiveIntervalYearMonth();``
break;
default: // 当类型都匹配不上的时候 抛出异常
throw new UDFArgumentTypeException(0, getFuncName()
+ " only takes INTERVAL_YEAR_MONTH and STRING_GROUP types, got " + inputTypes[i]);
}
return intervalYearMonth;
}
getIntervalYearMonthValue 方法对 输入的进行封装成 HiveIntervalYearMonth 当中有 getYear() 直接返回年份
protected final static int MONTHS_PER_YEAR = 12;
public int getYears() {
// 使用取整的方式进行获取年
return totalMonths / MONTHS_PER_YEAR;
}
public void set(int years, int months) {
this.totalMonths = months;
this.totalMonths += years * MONTHS_PER_YEAR;
}
当输入的是 字符串和TIMSTAMP类型的时候 走 getDateValue(arguments, 0, inputTypes, converters); 返回一个 date
// org.apache.hadoop.hive.ql.udf.generic.GenericUDF
protected Date getDateValue(DeferredObject[] arguments, int i, PrimitiveCategory[] inputTypes,
Converter[] converters) throws HiveException {
Object obj;
if ((obj = arguments[i].get()) == null) {
return null;
}
Date date;
switch (inputTypes[i]) {
case STRING:
case VARCHAR:
case CHAR:
String dateStr = converters[i].convert(obj).toString();
try {
// 转换date期类型
date = Date.valueOf(dateStr);
} catch (IllegalArgumentException e) {
date = null;
}
break;
case TIMESTAMP:
case DATE:
case TIMESTAMPLOCALTZ:
Object writableValue = converters[i].convert(obj);
date = ((DateWritableV2) writableValue).get();
break;
default:
throw new UDFArgumentTypeException(0, getFuncName()
+ " only takes STRING_GROUP and DATE_GROUP types, got " + inputTypes[i]);
}
return date;
}
其核心就是 date = Date.valueOf(dateStr); 转换成date类型
再次回到 evaluate(DeferredObject[] arguments) 方法中
// 根据java的 calendar 类型 进行求年
calendar.setTimeInMillis(date.toEpochMilli());
output.set(calendar.get(Calendar.YEAR));
总结
- 虽然函数的功能很少，但是底层代码逻辑写的还是较为复杂的，其中有大量校验逻辑，这也充分说明hive源码当中做了很多代码质量的工作。
- 再读源码的时候，可以学习到很大大佬写代码的风格与思考，看似简单的逻辑，其实要写的健壮性比较高的话，还是需要考虑很多的。
- 其中有很多地方不懂得地方可以debug或者自己写代码测试
