simd + io_uring_test

CMakeLists.txt

@@ -13,4 +13,4 @@ add_compile_options(-march=haswell)
 #add_compile_options("-mavx2")
 add_executable(FADAS src/main.cpp include/DataStructures/TimeSeries.h include/DataStructures/PqItemSeries.h src/DSTree/DSTreeNode.cpp include/DSTree/DSTreeNode.h include/DSTree/InsertedSeries.h include/DSTree/Sketch.h include/DSTree/PqItemNode.h include/DSTree/SplitInfo.h include/DSTree/INodeSegmentSplitPolicy.h src/DSTree/DSTreeNodeConstruction.cpp include/DSTree/DSTreeNodeConstruction.h src/DSTree/MeanNodeSegmentSplitPolicy.cpp include/DSTree/MeanNodeSegmentSplitPolicy.h src/DSTree/StdevNodeSegmentSplitPolicy.cpp include/DSTree/StdevNodeSegmentSplitPolicy.h src/Utils/FileUtil.cpp include/Utils/FileUtil.h src/Utils/TimeSeriesUtil.cpp include/Utils/TimeSeriesUtil.h src/Utils/SaxUtil.cpp include/Utils/SaxUtil.h include/DataStructures/PqItemIndex.h src/Utils/MathUtil.cpp include/Utils/MathUtil.h src/SearchEngine/ExactSearcher.cpp include/Searchers/ExactSearcher.h src/SearchEngine/DSTreeApproxSearcher.cpp include/Searchers/DSTreeApproxSearcher.h src/SearchEngine/DSTreeExactSearcher.cpp include/Searchers/DSTreeExactSearcher.h src/IndexConstruction/GraphConstruction.cpp include/DataStructures/GraphConstruction.h src/PqItemSeries.cpp src/DSTree/INodeSegmentSplitPolicy.cpp src/Expr/Recall.cpp include/Expr/Recall.h include/Const.h src/Expr/DataDistribution.cpp include/Expr/DataDistribution.h src/Tardis/TardisTreeNode.cpp include/Tardis/TardisTreeNode.h src/SearchEngine/TardisApproxSearch.cpp include/Searchers/TardisApproxSearch.h src/IndexConstruction/IPGNode.cpp include/DataStructures/IPGNode.h src/IndexConstruction/IPGPartition.cpp include/DataStructures/IPGPartition.h src/SearchEngine/IPGApproxSearcher.cpp include/Searchers/IPGApproxSearcher.h src/iSAX/iSAXNode.cpp include/DataStructures/iSAXNode.h src/SearchEngine/iSAXSearcher.cpp include/Searchers/iSAXSearcher.h src/Expr/RandDataGenerator.cpp include/Expr/RandDataGenerator.h src/Expr/DNATranslator.cpp include/Expr/DNATranslator.h src/IndexConstruction/IPGDataNode.cpp include/DataStructures/IPGDataNode.h src/Utils/INIReader.cpp include/Utils/INIReader.h src/Utils/ini.c include/DataStructures/OffsetDist.h src/Const.cpp src/IndexConstruction/FADASNode.cpp include/DataStructures/FADASNode.h src/SearchEngine/FADASSearcher.cpp include/Searchers/FADASSearcher.h src/IndexConstruction/FADASFuzzy.cpp src/Expr/ECGParser.cpp include/Expr/ECGParser.h src/TAR/TARGNode.cpp include/TAR/TARGNode.h src/TAR/TARLNode.cpp include/TAR/TARLNode.h src/SearchEngine/TARSearcher.cpp include/TAR/TARSearcher.h)
 #target_compile_options(FADAS PUBLIC "-mavx2")
-target_link_libraries(FADAS Threads::Threads libboost_serialization-mt-x64.a libboost_serialization-mt-x64.so libboost_serialization.so.1.76.0 ${GSL_LIBRARIES} ${GSLBLAS_LIBRARIES})
+target_link_libraries(FADAS Threads::Threads libboost_serialization-mt-x64.a libboost_serialization-mt-x64.so libboost_serialization.so.1.76.0 ${GSL_LIBRARIES} ${GSLBLAS_LIBRARIES} liburing.so)

config.ini

@@ -84,11 +84,14 @@ posidxfn = ../index-pos/rand/
 ;queryfn = /mnt/c/Series4Similarity_Search/rand/rand-1024-200.bin
 ;resfn = /mnt/c/Series4Similarity_Search/rand/rand-1024-200-50.bin
 ;fuzzyidxfn = /mnt/c/Series4Similarity_Search/rand/index-1024/
-fidxfn = /mnt/c/Series4Similarity_Search/rand/index/
+;fidxfn = /mnt/c/Series4Similarity_Search/rand/index/
+fidxfn = /home/wzy/data/rand/index/
 fuzzyidxfn = /mnt/c/Series4Similarity_Search/rand/fuzzy/
 datafn = /mnt/c/Series4Similarity_Search/rand/rand-256-100m.bin
-queryfn = /mnt/c/Series4Similarity_Search/rand/rand-256-200.bin
-resfn = /mnt/c/Series4Similarity_Search/rand/rand-256-200-50.bin
+;queryfn = /mnt/c/Series4Similarity_Search/rand/rand-256-200.bin
+;resfn = /mnt/c/Series4Similarity_Search/rand/rand-256-200-50.bin
+queryfn = /home/wzy/data/rand/rand-256-200.bin
+resfn = /home/wzy/data/rand/rand-256-200-50.bin
 dtwresfn = /mnt/c/Series4Similarity_Search/rand/rand-256-200-50-dtw2.bin
 dstreefn = /mnt/c/Series4Similarity_Search/ds-tree/rand/
 

include/DataStructures/FADASNode.h

@@ -187,7 +187,8 @@ class FADASNode {
                 float *query_reordered, int *ordering) const;
     void search_SIMD_reordered(int k, TimeSeries *queryTs, vector<PqItemSeries *> &heap, const string &index_dir,
                                float *query_reordered, int *ordering) const;
-    void search(int k, TimeSeries *queryTs, vector<PqItemSeries *> &heap, const string &index_dir) const;
+    void search_SIMD(int k, TimeSeries *queryTs, vector<PqItemSeries *> &heap, const string &index_dir) const;
+        void search(int k, TimeSeries *queryTs, vector<PqItemSeries *> &heap, const string &index_dir) const;
     void search(int k, TimeSeries *queryTs, vector<PqItemSeries *> &heap, const string &index_dir,
                 std::unordered_set<float *, createhash, isEqual> *hash_set) const;
     void search(int k, TimeSeries *queryTs, vector<PqItemSeries *> &heap, const string &index_dir,unordered_set<float*, createhash, isEqual>*hash_set,

include/Utils/SaxUtil.h

@@ -70,6 +70,10 @@ class SaxUtil {
 
     static double LowerBound_Paa_iSax_SIMD(const float *paa, const unsigned short *sax, const int* bits_cardinality, vector<int>&chosen_segs, int new_i);
 
+    static double LowerBound_Paa_iSax_SIMD(const float *paa, const unsigned short *sax, const int* bits_cardinality) ;
+
+        static double  minidist_paa_to_isax_rawa_SIMD(const float *paa, unsigned char*sax, unsigned char*sax_cardinalities);
+
     static double getMinDist1stLayer(const float *paa, int id);
 
     static int invSaxHead2FromSax(vector<int> *sax, int bitsCardinality, int segmentNum);

src/IndexConstruction/FADASNode.cpp

@@ -7,6 +7,7 @@
 #include <cmath>
 #include <bitset>
 #include <unordered_map>
+#include "liburing.h"
 #include "../../include/DataStructures/FADASNode.h"
 #include "../../include/Utils/FileUtil.h"
 #include "../../include/Utils/MathUtil.h"
@@ -205,31 +206,41 @@ void FADASNode::search_SIMD_reordered(int k, TimeSeries *queryTs, vector<PqItemS
     fclose(f);
 }
 
+#define URING
 void FADASNode::search_SIMD(int k, TimeSeries *queryTs, vector<PqItemSeries *> &heap, const string &index_dir) const{
     assert(!isInternalNode());
     double bsf = heap.size() < k? numeric_limits<double>::max() : heap[0]->dist;
     _search_num += size;
     string fn = index_dir+ getFileName();
     if(partition_id == -1)  fn += "_L";
-//    long fs = FileUtil::getFileSize(fn.c_str());
-//    int series_num = fs / Const::tsLengthBytes;
-//    assert(series_num == size);
 
-    FILE *f = fopen(fn.c_str(), "rb");
     struct timeval io{};
     Const::timer_start(&io);
     auto *ts = new float[size * Const::tsLength];
-//    for(int i=0;i<size;++i)
-//        fread(ts + i * Const::tsLength, sizeof(float), Const::tsLength, f);
+#ifndef URING
+    FILE *f = fopen(fn.c_str(), "rb");
     fread(ts, sizeof(float), size * Const::tsLength, f);
+#else
+    io_uring ring{};
+    auto ret = io_uring_queue_init(16, &ring, 0);
+    auto fd = open(fn.c_str(), O_RDONLY);
+    iovec vec{};
+    vec.iov_base = ts;
+    vec.iov_len = size * Const::tsLength * sizeof(float );
+    io_uring_sqe* sqe = io_uring_get_sqe(&ring);
+    io_uring_prep_read(sqe, fd, ts, size * Const::tsLength * sizeof(float ), 0);
+    ret = io_uring_submit(&ring);
+    io_uring_cqe* cqe;
+    ret = io_uring_wait_cqe(&ring, &cqe);
+#endif
     READ_TIME += Const::timer_end(&io);
 
     _search_num += size; ::layer = FADASNode::layer;
     for(int i=0;i<size;++i){
 //        struct timeval start{};
 //        Const::timer_start(&start);
 //        double dist = TimeSeriesUtil::euclideanDist(queryTs->ts, ts + i * Const::tsLength, Const::tsLength, bsf);
-        double dist = TimeSeriesUtil::euclideanDist_SIMD(query_reordered, ts + i * Const::tsLength, Const::tsLength, bsf, ordering);
+        double dist = TimeSeriesUtil::euclideanDist_SIMD(queryTs->ts, ts + i * Const::tsLength, Const::tsLength, bsf);
 //        DIST_CALC_TIME += Const::timer_end(&start);
 
         if(heap.size() < k){
@@ -250,7 +261,12 @@ void FADASNode::search_SIMD(int k, TimeSeries *queryTs, vector<PqItemSeries *> &
         if(s->needDeepCopy) s->copyData();
     }
     delete[]ts;
+#ifndef URING
     fclose(f);
+#else
+    close(fd);
+    io_uring_queue_exit(&ring);
+#endif
 }
 
 void FADASNode::search(int k, TimeSeries *queryTs, vector<PqItemSeries *> &heap, const string &index_dir) const{

src/SearchEngine/FADASSearcher.cpp

@@ -457,7 +457,8 @@ vector<PqItemSeries*>*FADASSearcher::exactSearchIdLevel(FADASNode* root, float *
             }
         }else{
             ++LOADED_NODE_CNT;
-            node->search_SIMD_reordered(k, queryTs, *heap, Const::fidxfn, query_reordered, ordering);
+//            node->search_SIMD_reordered(k, queryTs, *heap, Const::fidxfn, query_reordered, ordering);
+            node->search_SIMD(k,queryTs,*heap,Const::fidxfn);
             bsf = (*heap)[0]->dist;
         }
     }