Initial commit

.github/workflows/update-extractor.yml

@@ -0,0 +1,34 @@
+name: Update Extractor
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches:
+      - "main"
+    tags:
+      - "v*"
+
+jobs:
+  update_extractor:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+      - name: get latest tag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+        id: get-latest-tag
+        with:
+          fallback: latest
+      - name: Build the docker Docker image
+        run: |
+          docker build . -f Dockerfile.extractor --tag beclab/r4extractor:${{ steps.get-latest-tag.outputs.tag }}
+          docker push beclab/r4extractor:${{ steps.get-latest-tag.outputs.tag }}

.github/workflows/update-prerank.yml

@@ -0,0 +1,34 @@
+name: Update Prerank
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches:
+      - "main"
+    tags:
+      - "v*"
+
+jobs:
+  update_prerank:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+      - name: get latest tag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+        id: get-latest-tag
+        with:
+          fallback: latest
+      - name: Build the docker Docker image
+        run: |
+          docker build . -f Dockerfile.prerank --tag beclab/r4prerank:${{ steps.get-latest-tag.outputs.tag }}
+          docker push beclab/r4prerank:${{ steps.get-latest-tag.outputs.tag }}

.github/workflows/update-recall.yml

@@ -0,0 +1,34 @@
+name: Update Recall
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches:
+      - "main"
+    tags:
+      - "v*"
+
+jobs:
+  update_recall:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+      - name: get latest tag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+        id: get-latest-tag
+        with:
+          fallback: latest
+      - name: Build the docker Docker image
+        run: |
+          docker build . -f Dockerfile.recall --tag beclab/r4recall:${{ steps.get-latest-tag.outputs.tag }}
+          docker push beclab/r4recall:${{ steps.get-latest-tag.outputs.tag }}

.github/workflows/update_r4rank.yaml

@@ -0,0 +1,31 @@
+name: Update R4 Rank
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches:
+      - "main"
+    tags:
+      - "v*"
+
+jobs:
+  update_sdk:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+      - name: get latest tag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+        id: get-latest-tag
+        with:
+          fallback: latest
+      - name: Build the docker Docker image
+        run: |
+          docker build . -f Dockerfile.r4rank --tag beclab/r4rank:${{ steps.get-latest-tag.outputs.tag }}
+          docker push beclab/r4rank:${{ steps.get-latest-tag.outputs.tag }}

.github/workflows/update_r4train.yaml

@@ -0,0 +1,31 @@
+name: Update R4 TRAIN
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches:
+      - "main"
+    tags:
+      - "v*"
+
+jobs:
+  update_sdk:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+      - name: get latest tag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+        id: get-latest-tag
+        with:
+          fallback: latest
+      - name: Build the docker Docker image
+        run: |
+          docker build . -f Dockerfile.r4train --tag beclab/r4train:${{ steps.get-latest-tag.outputs.tag }}
+          docker push beclab/r4train:${{ steps.get-latest-tag.outputs.tag }}

.github/workflows/update_r4userembedding.yaml

@@ -0,0 +1,31 @@
+name: Update R4 User Embedding
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches:
+      - "main"
+    tags:
+      - "v*"
+
+jobs:
+  update_sdk:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+      - name: get latest tag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+        id: get-latest-tag
+        with:
+          fallback: latest
+      - name: Build the docker Docker image
+        run: |
+          docker build . -f Dockerfile.r4userembedding --tag beclab/r4userembedding:${{ steps.get-latest-tag.outputs.tag }}
+          docker push beclab/r4userembedding:${{ steps.get-latest-tag.outputs.tag }}

.gitignore

@@ -0,0 +1,18 @@
+*.rpm
+*.deb
+.DS_Store
+**/.DS_Store
+.vscode
+
+
+*.log
+.idea
+*.csv
+*.pdf
+*.xlsx
+*.dcm
+
+
+
+
+

Dockerfile.extractor

@@ -0,0 +1,16 @@
+FROM golang:alpine AS build
+
+RUN mkdir /extractorBuild
+COPY prerank-stages  /extractorBuild/prerank-stages
+
+WORKDIR /extractorBuild/prerank-stages
+RUN go mod download
+
+
+RUN cd /extractorBuild/prerank-stages/extractor/ && \
+    go build -o extractor_cmd  -ldflags="-s -w " main.go 
+
+FROM alpine:latest
+COPY --from=build /extractorBuild/prerank-stages/extractor/extractor_cmd /usr/bin/extractor_cmd
+
+ENTRYPOINT ["/usr/bin/extractor_cmd"]

Dockerfile.prerank

@@ -0,0 +1,16 @@
+FROM golang:alpine AS build
+
+RUN mkdir /prerankBuild
+COPY prerank-stages  /prerankBuild/prerank-stages
+
+WORKDIR /prerankBuild/prerank-stages
+RUN go mod download
+
+
+RUN cd /prerankBuild/prerank-stages/prerank/ && \
+    go build -o prerank_cmd  -ldflags="-s -w " main.go 
+
+FROM alpine:latest
+COPY --from=build /prerankBuild/prerank-stages/prerank/prerank_cmd /usr/bin/prerank_cmd
+COPY prerank-stages/config/  /usr/config/ 
+ENTRYPOINT ["/usr/bin/prerank_cmd"]

Dockerfile.r4rank

@@ -0,0 +1,117 @@
+FROM ubuntu:jammy as builder
+RUN mkdir -p /opt/rss-termius-v2-rank && \
+    mkdir -p /opt/xgboost && \
+    mkdir -p /opt/rank_model/
+RUN apt-get update && \
+    apt-get install build-essential -y && \
+    apt-get install cmake -y && \
+    apt install git -y && \
+    apt install curl -y && \
+    apt install pkg-config -y && \
+    apt-get install libssl-dev -y && \
+    apt-get install libcpprest-dev -y && \
+    cd /opt/xgboost && \
+    git clone --recursive https://github.com/dmlc/xgboost . && \
+    git checkout -b release_2.0.0 origin/release_2.0.0 && \
+    git submodule init && \
+    git submodule update && \
+    mkdir build && \
+    cd build && \
+    cmake ..  -DBUILD_STATIC_LIB=on && \
+    make -j$(nproc) && \
+    make install && \
+    cd /opt && \
+    rm -r xgboost 
+
+
+RUN cd /opt && \
+    curl -LJO https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz  && \
+    tar -xvzf googletest-1.14.0.tar.gz && \
+    cd /opt/googletest-1.14.0 && \
+    mkdir build && \
+    cd build && \
+    cmake .. && \
+    make -j$(nproc) && \
+    make install && \
+    cd /opt && \
+    rm -r googletest-1.14.0 && \
+    rm googletest-1.14.0.tar.gz
+
+
+RUN cd /opt && \
+    curl -LJO https://github.com/mongodb/mongo-c-driver/releases/download/1.24.0/mongo-c-driver-1.24.0.tar.gz && \
+    tar -xvzf mongo-c-driver-1.24.0.tar.gz && \
+    cd mongo-c-driver-1.24.0/build && \
+    cmake .. && \
+    make -j$(nproc) && \
+    make install && \
+    cd /opt && \
+    rm mongo-c-driver-1.24.0.tar.gz && \
+    rm -r mongo-c-driver-1.24.0
+
+RUN cd /opt && \
+    curl -LJO  https://github.com/mongodb/mongo-cxx-driver/releases/download/r3.8.1/mongo-cxx-driver-r3.8.1.tar.gz && \
+    tar -xvzf  mongo-cxx-driver-r3.8.1.tar.gz && \
+    cd mongo-cxx-driver-r3.8.1/build && \
+    cmake -DCMAKE_INSTALL_PREFIX=/usr/local .. && \
+    make -j$(nproc) && \
+    make install && \
+    cd /opt && \
+    rm mongo-cxx-driver-r3.8.1.tar.gz && \
+    rm -r mongo-cxx-driver-r3.8.1
+
+RUN cd /opt && \
+    curl -LO https://github.com/google/sentencepiece/archive/refs/tags/v0.1.99.tar.gz && \
+    tar -xvzf v0.1.99.tar.gz && \
+    cd sentencepiece-0.1.99 && \
+    mkdir build && \
+    cd build  && \
+    cmake .. && \
+    make -j$(nproc) && \
+    make install && \
+    cd /opt && \
+    rm -r sentencepiece-0.1.99 && \
+    rm v0.1.99.tar.gz
+
+RUN cd /opt && \
+    mkdir -p /usr/local/include/cppitertools && \
+    curl -LO https://github.com/ryanhaining/cppitertools/archive/refs/tags/v2.1.tar.gz && \
+    tar xzvf v2.1.tar.gz && \
+    cd cppitertools-2.1/ && \
+    cp *.hpp /usr/local/include/cppitertools && \
+    cp -r internal /usr/local/include/cppitertools && \
+    cd /opt && \
+    rm  v2.1.tar.gz  && \
+    rm -r cppitertools-2.1
+
+ADD train-rank/src /opt/rss-terminus-v2-rank/src
+ADD train-rank/test /opt/rss-terminus-v2-rank/test
+COPY train-rank/CMakeLists.txt /opt/rss-terminus-v2-rank/CMakeLists.txt
+RUN cd /opt/rss-terminus-v2-rank && \
+    mkdir build && \
+    cd build && \
+    cmake .. && \
+    make && \
+    cp bertv2rank /usr/local/bin && \
+    cd /opt && \
+    rm -r rss-termius-v2-rank
+
+FROM ubuntu:jammy
+# FROM alpine
+
+
+COPY --from=builder /usr/local/bin/bertv2rank /usr/local/bin/bertv2rank
+COPY  --from=builder /usr/local/lib/libmongocxx.so._noabi /usr/local/lib/libmongocxx.so._noabi
+COPY  --from=builder /usr/local/lib/libbsoncxx.so._noabi  /usr/local/lib/libbsoncxx.so._noabi
+COPY  --from=builder  /usr/local/lib/libmongoc-1.0.so.0  /usr/local/lib/libmongoc-1.0.so.0
+COPY   --from=builder  /usr/local/lib/libbson-1.0.so.0  /usr/local/lib/libbson-1.0.so.0
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libgomp.so.1 /usr/lib/x86_64-linux-gnu/libgomp.so.1
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10  /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1  /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libicuuc.so.70  /usr/lib/x86_64-linux-gnu/libicuuc.so.70 
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libicudata.so.70 /usr/lib/x86_64-linux-gnu/libicudata.so.70
+RUN mkdir /opt/rank_model/
+CMD ["/usr/local/bin/bertv2rank"]
+# ENTRYPOINT ["tail", "-f", "/dev/null"]