more prep for supporting strings

yutiansut · Feb 10, 2015 · eb1956b · eb1956b
1 parent 23b83b1
commit eb1956b
Show file tree

Hide file tree

Showing 9 changed files with 18 additions and 232 deletions.
diff --git a/Catalog/ColumnDescriptor.h b/Catalog/ColumnDescriptor.h
@@ -24,6 +24,8 @@ struct ColumnDescriptor {
     ColumnDescriptor(const int tableId, const int columnId, const std::string &columnName, const SQLTypeInfo columnType, const EncodingType compression, const int comp_param = 0): tableId(tableId), columnId(columnId), columnName(columnName),columnType(columnType),compression(compression),comp_param(comp_param) {
     } 
 
+		inline bool is_varlen() const { return IS_STRING(columnType.type) && compression != kENCODING_DICT; }
+
 		int getStorageSize() const {
 			switch (columnType.type) {
 				case kBOOLEAN:

diff --git a/Catalog/mk_test.sh b/Catalog/mk_test.sh
diff --git a/Chunk/Chunk.cpp b/Chunk/Chunk.cpp
@@ -7,17 +7,24 @@
 
 namespace Chunk_NS {
 	Chunk
-	Chunk::getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId, const size_t numBytes) {
+	Chunk::getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId, const size_t numBytes, const size_t numElems) {
 			Chunk chunk(cd);
-			chunk.getChunkBuffer(data_mgr, key, memoryLevel, deviceId, numBytes);
+			chunk.getChunkBuffer(data_mgr, key, memoryLevel, deviceId, numBytes, numElems);
 			return chunk;
 		}
 
 	void 
-	Chunk::getChunkBuffer(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int device_id, const size_t num_bytes)
+	Chunk::getChunkBuffer(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int device_id, const size_t num_bytes, const size_t num_elems)
 	{
-		// @TODO add logic here to handle string case
-		buffer = data_mgr->getChunkBuffer(key, mem_level, device_id, num_bytes);
+		if (column_desc->is_varlen()) {
+			ChunkKey subKey = key;
+			subKey.push_back(1); // 1 for the main buffer
+			buffer = data_mgr->getChunkBuffer(subKey, mem_level, device_id, num_bytes);
+			subKey.pop_back();
+			subKey.push_back(2); // 2 for the index buffer
+			index_buf = data_mgr->getChunkBuffer(subKey, mem_level, device_id, (num_elems + 1) * sizeof(int32_t)); // always record n+1 offsets so string length can be calculated
+		} else
+			buffer = data_mgr->getChunkBuffer(key, mem_level, device_id, num_bytes);
 	}
 
 	void

diff --git a/Chunk/Chunk.h b/Chunk/Chunk.h
@@ -50,8 +50,8 @@ namespace Chunk_NS {
 			ChunkIter begin_iterator(int start_idx, int skip) const;
 			ChunkMetadata appendData(DataBlockPtr &srcData, const size_t numAppendElems);
 			void createChunkBuffer(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId = 0);
-			void getChunkBuffer(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId = 0, const size_t num_bytes = 0);
-			static Chunk getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes);
+			void getChunkBuffer(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId = 0, const size_t num_bytes = 0, const size_t num_elems = 0);
+			static Chunk getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems = 0);
 
 		// protected:
 			AbstractBuffer *get_buffer() const { return buffer; }

diff --git a/DataMgr/tests/DataMgrTest.cpp b/DataMgr/tests/DataMgrTest.cpp
diff --git a/DataMgr/tests/build_datamgr_test.linux.sh b/DataMgr/tests/build_datamgr_test.linux.sh
diff --git a/DataMgr/tests/build_datamgr_test.mac.sh b/DataMgr/tests/build_datamgr_test.mac.sh
diff --git a/QueryEngine/Execute.cpp b/QueryEngine/Execute.cpp
@@ -957,7 +957,8 @@ std::vector<ResultRow> Executor::executeAggScanPlan(
           chunk_key,
           memory_level,
           fragment.deviceIds[static_cast<int>(memory_level)],
-          chunk_meta_it->second.numBytes);
+          chunk_meta_it->second.numBytes,
+					chunk_meta_it->second.numElements);
         auto ab = chunk.get_buffer();
         CHECK(ab->getMemoryPtr());
         auto it = global_to_local_col_ids_.find(col_id);

diff --git a/Shared/sqltypes.h b/Shared/sqltypes.h
@@ -35,7 +35,6 @@ enum SQLTypes {
 
 #define IS_NUMBER(T) (((T) == kINT) || ((T) == kSMALLINT) || ((T) == kDOUBLE) || ((T) == kFLOAT) || ((T) == kBIGINT) || ((T) == kNUMERIC) || ((T) == kDECIMAL))
 #define IS_STRING(T) (((T) == kTEXT) || ((T) == kVARCHAR) || ((T) == kCHAR))
-#define IS_VARLEN(T) IS_STRING(T)
 
 typedef union {
 	bool boolval;