Merge pull request #101 from janhq/41-feat-batch-inference-for-nitro

Merge batching inference to nitro + fix cuda on windows issues (shouldn't use shared libs)

Author: tikikun

.github/workflows/build.yml

@@ -292,8 +292,6 @@ jobs:
         run: |
           robocopy build_deps\_install\bin .\build\Release zlib.dll
           robocopy build\bin\Release .\build\Release llama.dll
-          robocopy ext_libs .\build\Release libcrypto-3-x64.dll
-          robocopy ext_libs .\build\Release libssl-3-x64.dll
           7z a nitro.zip .\build\Release\*
           cd .\build\Release
           ..\..\.github\scripts\e2e-test-windows.bat .\nitro.exe ${{ env.MODEL_URL }}
@@ -343,7 +341,7 @@ jobs:
           cmake --build ./build_deps/nitro_deps --config Release
           mkdir -p build
           cd build
-          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUBLAS=ON -DBUILD_SHARED_LIBS=ON
+          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUBLAS=ON
           cmake --build . --config Release -j 4
 
       - name: Pack artifacts
@@ -354,8 +352,6 @@ jobs:
           echo %PATH%
           robocopy build_deps\_install\bin .\build\Release zlib.dll
           robocopy build\bin\Release .\build\Release llama.dll
-          robocopy ext_libs .\build\Release libcrypto-3-x64.dll
-          robocopy ext_libs .\build\Release libssl-3-x64.dll
           7z a nitro.zip .\build\Release\*
           cd .\build\Release
           ..\..\.github\scripts\e2e-test-windows.bat .\nitro.exe ${{ env.MODEL_URL }}

CMakeLists.txt

@@ -16,6 +16,9 @@ else()
   set(CMAKE_CXX_STANDARD 14)
 endif()
 
+# llama cpp server need llava example to work, this is for llama cpp server
+set(LLAMA_BUILD_EXAMPLES ON)
+
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(OPENSSL_USE_STATIC_LIBS TRUE)
@@ -48,7 +51,7 @@ add_executable(${PROJECT_NAME} main.cc)
 #
 # and comment out the following lines
 find_package(Drogon CONFIG REQUIRED)
-target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon common llama
+target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon common llama clip
                                               ${CMAKE_THREAD_LIBS_INIT})
 
 # ##############################################################################