当前位置：源码 > Java源码 >

Java源码示例：org.apache.kylin.cube.model.HBaseColumnDesc

示例1

public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

        cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
        qBytes = Bytes.toBytes(colDesc.getQualifier());
        timestamp = 0; // use 0 for timestamp

        refIndex = colDesc.getMeasureIndex();
        refMeasures = colDesc.getMeasures();

        codec = new BufferedMeasureCodec(refMeasures);
        colValues = new Object[refMeasures.length];

        isFullCopy = true;
        List<MeasureDesc> measures = cubeDesc.getMeasures();
        for (int i = 0; i < measures.size(); i++) {
            if (refIndex.length <= i || refIndex[i] != i)
                isFullCopy = false;
        }
    }

示例2

@Override
protected void doSetup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    CubeManager cubeMgr = CubeManager.getInstance(config);
    cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();

    inputCodec = new MeasureCodec(cubeDesc.getMeasures());
    inputMeasures = new Object[cubeDesc.getMeasures().size()];
    keyValueCreators = Lists.newArrayList();

    for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
            keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
        }
    }
}

示例3

/**
 * prune untouched hbase columns
 */
protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
    List<Pair<byte[], byte[]>> result = Lists.newArrayList();

    int colBlkIndex = 1;
    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
                result.add(Pair.newPair(byteFamily, byteQualifier));
            }
            colBlkIndex++;
        }
    }

    return result;
}

示例4

/**
 * for each selected hbase column, it might contain values of multiple GT columns.
 * The mapping should be passed down to storage
 */
protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) {

    List<List<Integer>> ret = Lists.newArrayList();

    int colBlkIndex = 1;
    int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount();

    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                int[] metricIndexes = hbaseColDesc.getMeasureIndex();
                Integer[] gtIndexes = new Integer[metricIndexes.length];
                for (int i = 0; i < gtIndexes.length; i++) {
                    gtIndexes[i] = metricIndexes[i] + metricOffset;
                }
                ret.add(Arrays.asList(gtIndexes));
            }
            colBlkIndex++;
        }
    }

    Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1);
    return ret;
}

示例5

public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

        cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
        qBytes = Bytes.toBytes(colDesc.getQualifier());
        timestamp = 0; // use 0 for timestamp

        refIndex = colDesc.getMeasureIndex();
        refMeasures = colDesc.getMeasures();

        codec = new BufferedMeasureCodec(refMeasures);
        colValues = new Object[refMeasures.length];

        isFullCopy = true;
        List<MeasureDesc> measures = cubeDesc.getMeasures();
        for (int i = 0; i < measures.size(); i++) {
            if (refIndex.length <= i || refIndex[i] != i)
                isFullCopy = false;
        }
    }

示例6

@Override
protected void doSetup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    CubeManager cubeMgr = CubeManager.getInstance(config);
    cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();

    inputCodec = new MeasureCodec(cubeDesc.getMeasures());
    inputMeasures = new Object[cubeDesc.getMeasures().size()];
    keyValueCreators = Lists.newArrayList();

    for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
            keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
        }
    }
}

示例7

/**
 * prune untouched hbase columns
 */
protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
    List<Pair<byte[], byte[]>> result = Lists.newArrayList();

    int colBlkIndex = 1;
    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
                result.add(Pair.newPair(byteFamily, byteQualifier));
            }
            colBlkIndex++;
        }
    }

    return result;
}

示例8

/**
 * for each selected hbase column, it might contain values of multiple GT columns.
 * The mapping should be passed down to storage
 */
protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) {

    List<List<Integer>> ret = Lists.newArrayList();

    int colBlkIndex = 1;
    int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount();

    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                int[] metricIndexes = hbaseColDesc.getMeasureIndex();
                Integer[] gtIndexes = new Integer[metricIndexes.length];
                for (int i = 0; i < gtIndexes.length; i++) {
                    gtIndexes[i] = metricIndexes[i] + metricOffset;
                }
                ret.add(Arrays.asList(gtIndexes));
            }
            colBlkIndex++;
        }
    }

    Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1);
    return ret;
}

示例9

@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());
    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());

    CubeManager cubeMgr = CubeManager.getInstance(config);
    cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();

    inputCodec = new MeasureCodec(cubeDesc.getMeasures());
    inputMeasures = new Object[cubeDesc.getMeasures().size()];
    keyValueCreators = Lists.newArrayList();

    for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
            keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
        }
    }
}

示例10

public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

            cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
            qBytes = Bytes.toBytes(colDesc.getQualifier());
            timestamp = System.currentTimeMillis();

            List<MeasureDesc> measures = cubeDesc.getMeasures();
            String[] measureNames = getMeasureNames(cubeDesc);
            String[] refs = colDesc.getMeasureRefs();

            refIndex = new int[refs.length];
            refMeasures = new MeasureDesc[refs.length];
            for (int i = 0; i < refs.length; i++) {
                refIndex[i] = indexOf(measureNames, refs[i]);
                refMeasures[i] = measures.get(refIndex[i]);
            }

            codec = new MeasureCodec(refMeasures);
            colValues = new Object[refs.length];

            isFullCopy = true;
            for (int i = 0; i < measures.size(); i++) {
                if (refIndex.length <= i || refIndex[i] != i)
                    isFullCopy = false;
            }
        }

示例11

private void initMeasureReferenceToColumnFamily() {
    if (measures == null || measures.size() == 0)
        return;

    Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>();
    for (MeasureDesc m : measures)
        measureCache.put(m.getName(), m);

    for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc c : cf.getColumns()) {
            MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length];
            for (int i = 0; i < c.getMeasureRefs().length; i++) {
                measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]);
            }
            c.setMeasures(measureDescs);
            c.setColumnFamilyName(cf.getName());
        }
    }
}

示例12

public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
    this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
    this.projectionIndex = new BitSet();
    this.measures = hbaseColumn.getMeasures();
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}

示例13

@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    Long count = new Long(2);
    Long item_count = new Long(100);
    ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasure(aggrFunc);
        rowValueDecoder.setProjectIndex(index);
    }

    rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
    Object[] measureValues = rowValueDecoder.getValues();
    //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
    assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
}

示例14

@Test(expected = IllegalArgumentException.class)
public void testError() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    Long item_count = new Long(100);
    codec.encode(new Object[] { sum, min, max, count, item_count });

}

示例15

@Test
public void testBadInit14() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    String measureInfoForTransCnt = measureForTransCnt.toString();
    thrown.expectMessage(
            "measure (" + measureInfoForTransCnt + ") does not exist in column family, or measure duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}

示例16

@Test
public void testBadInit15() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(
            new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}

示例17

private void validateColumnFamily(CubeDesc cubeDesc) {
    Set<String> columnFamilyMetricsSet = Sets.newHashSet();
    for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) {
            for (String columnName : hBaseColumnDesc.getMeasureRefs()) {
                columnFamilyMetricsSet.add(columnName);
            }
        }
    }
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (!columnFamilyMetricsSet.contains(measureDesc.getName())) {
            throw new BadRequestException("column family lack measure:" + measureDesc.getName());
        }
    }
    if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) {
        throw new BadRequestException(
                "the number of input measure and the number of measure defined in cubedesc are not consistent");
    }

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
        String encodingName = (String) encodingConf[0];
        String[] encodingArgs = (String[]) encodingConf[1];

        if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs,
                rowKeyColDesc.getEncodingVersion())) {
            throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
        }
    }
}

示例18

public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
    this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
    this.projectionIndex = new BitSet();
    this.measures = hbaseColumn.getMeasures();
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}

示例19

@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    Long count = new Long(2);
    Long item_count = new Long(100);
    ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasure(aggrFunc);
        rowValueDecoder.setProjectIndex(index);
    }

    rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
    Object[] measureValues = rowValueDecoder.getValues();
    //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
    assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
}

示例20

@Test(expected = IllegalArgumentException.class)
public void testError() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    Long item_count = new Long(100);
    codec.encode(new Object[] { sum, min, max, count, item_count });

}

示例21

@Test
public void testBadInit14() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    String measureInfoForTransCnt = measureForTransCnt.toString();
    thrown.expectMessage(
            "measure (" + measureInfoForTransCnt + ") does not exist in column family, or measure duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}

示例22

@Test
public void testBadInit15() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(
            new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}

示例23

private void validateColumnFamily(CubeDesc cubeDesc) {
    Set<String> columnFamilyMetricsSet = Sets.newHashSet();
    for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) {
            for (String columnName : hBaseColumnDesc.getMeasureRefs()) {
                columnFamilyMetricsSet.add(columnName);
            }
        }
    }
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (!columnFamilyMetricsSet.contains(measureDesc.getName())) {
            throw new BadRequestException("column family lack measure:" + measureDesc.getName());
        }
    }
    if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) {
        throw new BadRequestException(
                "the number of input measure and the number of measure defined in cubedesc are not consistent");
    }

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
        String encodingName = (String) encodingConf[0];
        String[] encodingArgs = (String[]) encodingConf[1];

        if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs,
                rowKeyColDesc.getEncodingVersion())) {
            throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
        }
    }
}

示例24

public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.projectionIndex = new BitSet();
    this.names = new ArrayList<String>();
    this.measures = hbaseColumn.getMeasures();
    for (MeasureDesc measure : measures) {
        this.names.add(measure.getFunction().getRewriteFieldName());
    }
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}

示例25

@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];

    MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
    codec.encode(new Object[] { sum, min, max, count }, buf);

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasureIndex(aggrFunc);
        rowValueDecoder.setIndex(index);
    }

    rowValueDecoder.decode(valueBytes);
    List<String> measureNames = rowValueDecoder.getNames();
    Object[] measureValues = rowValueDecoder.getValues();

    assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString());
    assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues));
}

示例26

private Scan buildScan(HBaseKeyRange keyRange) {
    Scan scan = new Scan();
    scan.setCaching(SCAN_CACHE);
    scan.setCacheBlocks(true);
    scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
    for (RowValueDecoder valueDecoder : this.rowValueDecoders) {
        HBaseColumnDesc hbaseColumn = valueDecoder.getHBaseColumn();
        byte[] byteFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
        byte[] byteQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
        scan.addColumn(byteFamily, byteQualifier);
    }
    scan.setStartRow(keyRange.getStartKey());
    scan.setStopRow(keyRange.getStopKey());
    return scan;
}

示例27

private void translateResult(Result res, Tuple tuple) throws IOException {
    // groups
    byte[] rowkey = res.getRow();
    rowKeyDecoder.decode(rowkey);
    List<TblColRef> columns = rowKeyDecoder.getColumns();
    List<String> dimensionNames = rowKeyDecoder.getNames(context.getAliasMap());
    List<String> dimensionValues = rowKeyDecoder.getValues();
    for (int i = 0; i < dimensionNames.size(); i++) {
        TblColRef column = columns.get(i);
        if (!tuple.hasColumn(column)) {
            continue;
        }
        tuple.setDimensionValue(dimensionNames.get(i), dimensionValues.get(i));
    }

    // derived
    for (IDerivedColumnFiller filler : tupleInfo.getDerivedColumnFillers()) {
        filler.fillDerivedColumns(dimensionValues, tuple);
    }

    // aggregations
    for (RowValueDecoder rowValueDecoder : this.rowValueDecoders) {
        HBaseColumnDesc hbaseColumn = rowValueDecoder.getHBaseColumn();
        String columnFamily = hbaseColumn.getColumnFamilyName();
        String qualifier = hbaseColumn.getQualifier();
        // FIXME: avoidable bytes array creation, why not use res.getValueAsByteBuffer directly?
        byte[] valueBytes = res.getValue(Bytes.toBytes(columnFamily), Bytes.toBytes(qualifier));
        rowValueDecoder.decode(valueBytes);
        List<String> measureNames = rowValueDecoder.getNames();
        Object[] measureValues = rowValueDecoder.getValues();
        BitSet projectionIndex = rowValueDecoder.getProjectionIndex();
        for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) {
            tuple.setMeasureValue(measureNames.get(i), measureValues[i]);
        }
    }
}

示例28

private static HCol buildHCol(HBaseColumnDesc desc) {
    byte[] family = Bytes.toBytes(desc.getColumnFamilyName());
    byte[] qualifier = Bytes.toBytes(desc.getQualifier());
    MeasureDesc[] measures = desc.getMeasures();

    String[] funcNames = new String[measures.length];
    String[] dataTypes = new String[measures.length];

    for (int i = 0; i < measures.length; i++) {
        funcNames[i] = measures[i].getFunction().getExpression();
        dataTypes[i] = measures[i].getFunction().getReturnType();
    }

    return new HCol(family, qualifier, funcNames, dataTypes);
}

示例29

private List<RowValueDecoder> translateAggregation(HBaseMappingDesc hbaseMapping, Collection<FunctionDesc> metrics, //
        StorageContext context) {
    Map<HBaseColumnDesc, RowValueDecoder> codecMap = Maps.newHashMap();
    for (FunctionDesc aggrFunc : metrics) {
        Collection<HBaseColumnDesc> hbCols = hbaseMapping.findHBaseColumnByFunction(aggrFunc);
        if (hbCols.isEmpty()) {
            throw new IllegalStateException("can't find HBaseColumnDesc for function " + aggrFunc.getFullExpression());
        }
        HBaseColumnDesc bestHBCol = null;
        int bestIndex = -1;
        for (HBaseColumnDesc hbCol : hbCols) {
            bestHBCol = hbCol;
            bestIndex = hbCol.findMeasureIndex(aggrFunc);
            MeasureDesc measure = hbCol.getMeasures()[bestIndex];
            // criteria for holistic measure: Exact Aggregation && Exact Cuboid
            if (measure.isHolisticCountDistinct() && context.isExactAggregation()) {
                logger.info("Holistic count distinct chosen for " + aggrFunc);
                break;
            }
        }

        RowValueDecoder codec = codecMap.get(bestHBCol);
        if (codec == null) {
            codec = new RowValueDecoder(bestHBCol);
            codecMap.put(bestHBCol, codec);
        }
        codec.setIndex(bestIndex);
    }
    return new ArrayList<RowValueDecoder>(codecMap.values());
}

示例30

public HBaseColumnDesc getHBaseColumn() {
    return hbaseColumn;
}

Java源码示例：org.apache.kylin.cube.model.HBaseColumnDesc

微信关注