Skip to content

Commit

Permalink
Fixed data loading performance issue
Browse files Browse the repository at this point in the history
  • Loading branch information
kumarvishal09 authored and jackylk committed Nov 18, 2016
1 parent 2f24b52 commit 9a3b221
Showing 1 changed file with 6 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,10 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
private GenericDataType[] complexTypes;

private DirectDictionaryGenerator[] directDictionaryGenerators;

/**
* dimension column ids
*/
private String[] dimensionColumnIds;
/**
* Constructor
*
Expand Down Expand Up @@ -350,7 +353,7 @@ public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws K
if (null != getInputRowMeta()) {
data.setOutputRowMeta((RowMetaInterface) getInputRowMeta().clone());
}

this.dimensionColumnIds = meta.getDimensionColumnIds();
ColumnsInfo columnsInfo = new ColumnsInfo();
columnsInfo.setDims(meta.dims);
columnsInfo.setDimColNames(meta.dimColNames);
Expand All @@ -373,7 +376,7 @@ public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws K
columnsInfo.setPrimaryKeyMap(meta.getPrimaryKeyMap());
columnsInfo.setMeasureColumns(meta.measureColumn);
columnsInfo.setComplexTypesMap(meta.getComplexTypes());
columnsInfo.setDimensionColumnIds(meta.getDimensionColumnIds());
columnsInfo.setDimensionColumnIds(this.dimensionColumnIds);
columnsInfo.setColumnSchemaDetailsWrapper(meta.getColumnSchemaDetailsWrapper());
columnsInfo.setColumnProperties(meta.getColumnPropertiesMap());
updateBagLogFileName();
Expand Down Expand Up @@ -945,7 +948,6 @@ private Object[] populateOutputRow(Object[] r) throws KettleException {
int inputColumnsSize = metaColumnNames.length;
boolean isGenerated = false;
int generatedSurrogate = -1;
String[] dimensionColumnIds = meta.getDimensionColumnIds();

//If CSV Exported from DB and we enter one row down then that row become empty.
// In that case it will have first value empty and other values will be null
Expand Down

0 comments on commit 9a3b221

Please sign in to comment.