Skip to content

Commit

Permalink
add RoaringBitmap
Browse files Browse the repository at this point in the history
Signed-off-by: sylar-yin <564628276@qq.com>
  • Loading branch information
sylar-yin committed Sep 26, 2019
1 parent e6c8773 commit f1efd0a
Show file tree
Hide file tree
Showing 11 changed files with 21,395 additions and 35 deletions.
6 changes: 5 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
include (cmake/utils.cmake)

set(CMAKE_VERBOSE_MAKEFILE ON)
set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -rdynamic -O0 -fPIC -ggdb -std=c++11 -Wall -Wno-deprecated -Werror -Wno-unused-function -Wno-builtin-macro-redefined -Wno-deprecated-declarations")
set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -rdynamic -O3 -fPIC -ggdb -std=c++11 -Wall -Wno-deprecated -Werror -Wno-unused-function -Wno-builtin-macro-redefined -Wno-deprecated-declarations")
set(CMAKE_C_FLAGS "$ENV{CXXFLAGS} -rdynamic -O3 -fPIC -ggdb -std=c11 -Wall -Wno-deprecated -Werror -Wno-unused-function -Wno-builtin-macro-redefined -Wno-deprecated-declarations")

include_directories(.)
include_directories(/apps/sylar/include)
Expand Down Expand Up @@ -57,6 +58,8 @@ set(LIB_SRC
sylar/db/redis.cc
sylar/db/sqlite3.cc
sylar/ds/bitmap.cc
sylar/ds/roaring_bitmap.cc
sylar/ds/roaring.c
sylar/email/email.cc
sylar/email/smtp.cc
sylar/env.cc
Expand Down Expand Up @@ -142,6 +145,7 @@ set(LIBS
${OPENSSL_LIBRARIES}
${PROTOBUF_LIBRARIES}
event
jemalloc
hiredis_vip
mysqlclient_r
zookeeper_mt
Expand Down
2 changes: 1 addition & 1 deletion bin/conf/server.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
servers:
- address: ["0.0.0.0:8090", "127.0.0.1:0", "/tmp/test.sock"]
- address: ["0.0.0.0:8090", "127.0.0.1:8091", "/tmp/test.sock"]
keepalive: 1
timeout: 1000
name: sylar/1.1
Expand Down
2 changes: 1 addition & 1 deletion bin/conf/worker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ workers:
http_io:
thread_num: 1
accept:
thread_num: 1
thread_num: 2
worker:
thread_num: 8
notify:
Expand Down
66 changes: 36 additions & 30 deletions sylar/ds/bitmap.cc
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ void Bitmap::writeTo(sylar::ByteArray::ptr ba) const {
ba->writeFuint8(m_compress);
ba->writeFuint32(m_size);
ba->writeFuint32(m_dataSize);
ba->write((const char*)m_data, m_dataSize);
ba->write((const char*)m_data, m_dataSize * sizeof(base_type));
}

bool Bitmap::readFrom(sylar::ByteArray::ptr ba) {
Expand All @@ -76,7 +76,7 @@ bool Bitmap::readFrom(sylar::ByteArray::ptr ba) {
free(m_data);
}
m_data = (base_type*)malloc(m_dataSize * sizeof(base_type));
ba->read((char*)m_data, m_dataSize);
ba->read((char*)m_data, m_dataSize * sizeof(base_type));
return true;
} catch(...) {
}
Expand Down Expand Up @@ -333,7 +333,7 @@ Bitmap::ptr Bitmap::compress() const{

base_type* data = (base_type*)malloc(m_dataSize * sizeof(base_type));
bool cur_value = false;
uint32_t cur_count = 0;
uint64_t cur_count = 0;
uint32_t dst_cur_pos = 0;
for(uint32_t i = 0; i < m_dataSize; ++i) {
base_type cur = m_data[i];
Expand Down Expand Up @@ -507,7 +507,7 @@ void Bitmap::foreach(std::function<bool(uint32_t)> cb) {
}
for(uint32_t i = max_size * U64_DIV_BASE;
i < m_dataSize; ++i) {
tmp= m_data[i];
tmp = m_data[i];
if(!tmp) {
cur_pos += VALUE_SIZE;
} else {
Expand Down Expand Up @@ -633,7 +633,7 @@ void Bitmap::listPosAsc(std::vector<uint32_t>& pos) {
}
}
for(uint32_t i = max_size * U64_DIV_BASE; i < m_dataSize; ++i) {
tmp= m_data[i];
tmp = m_data[i];
if(!tmp) {
cur_pos += VALUE_SIZE;
} else {
Expand Down Expand Up @@ -869,36 +869,42 @@ Bitmap::iterator_base::ptr Bitmap::rbegin_new() {
}

uint32_t Bitmap::getCount() const {
uint32_t len = m_dataSize / U64_DIV_BASE;// - (m_dataSize % 8 == 0 ? 1 : 0);
uint32_t count = 0;
uint32_t cur_pos = 0;
for(uint32_t i = 0; i < len; ++i) {
uint64_t tmp = ((uint64_t*)(m_data))[i];
for(; tmp; ++count) {
tmp &= tmp - 1;
if(!m_compress) {
uint32_t len = m_dataSize / U64_DIV_BASE;// - (m_dataSize % 8 == 0 ? 1 : 0);
uint32_t count = 0;
uint32_t cur_pos = 0;
for(uint32_t i = 0; i < len; ++i) {
uint64_t tmp = ((uint64_t*)(m_data))[i];
count += count_bits(tmp);
cur_pos += U64_VALUE_SIZE;
}
cur_pos += U64_VALUE_SIZE;
}
for(uint32_t i = len * U64_DIV_BASE; i < m_dataSize; ++i) {
base_type tmp= m_data[i];
if(tmp) {
for(uint32_t n = 0; n < VALUE_SIZE && cur_pos < m_size; ++n, ++cur_pos) {
if(tmp & POS[n]) { //(1UL << n)) {
++count;
for(uint32_t i = len * U64_DIV_BASE; i < m_dataSize; ++i) {
base_type tmp = m_data[i];
if(tmp) {
for(uint32_t n = 0; n < VALUE_SIZE && cur_pos < m_size; ++n, ++cur_pos) {
if(tmp & POS[n]) { //(1UL << n)) {
++count;
}
}
} else {
cur_pos += VALUE_SIZE;
}
}
return count;
} else {
uint32_t count = 0;
for(uint32_t i = 0; i < m_dataSize; ++i) {
base_type tmp = m_data[i];
if(tmp & COMPRESS_MASK) {
if(tmp & VALUE_MASK) {
count += tmp & COUNT_MASK;
}
} else {
count += count_bits(tmp);
}
} else {
cur_pos += VALUE_SIZE;
}
//if(tmp) {
// for(uint32_t n = 0; n < 8 && cur_pos < m_size; ++n, ++cur_pos) {
// if(tmp & (1UL << n)) {
// ++count;
// }
// }
//}
return count;
}
return count;
}

}
Expand Down
2 changes: 1 addition & 1 deletion sylar/ds/bitmap.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ namespace ds {
#define BITMAP_TYPE_UINT64 4

#ifndef BITMAP_TYPE
#define BITMAP_TYPE BITMAP_TYPE_UINT8
#define BITMAP_TYPE BITMAP_TYPE_UINT16
#endif

class Bitmap {
Expand Down
Loading

0 comments on commit f1efd0a

Please sign in to comment.